diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 62f2e5bb..bb595602 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,9 +9,9 @@ on: workflow_dispatch: inputs: version_tag: - description: 'Version tag (e.g. 2.4.1)' + description: 'Version tag (e.g. 2.4.2)' required: true - default: '2.4.1' + default: '2.4.2' jobs: build-and-push: diff --git a/api/downloads.py b/api/downloads.py index 2f087874..69a9846d 100644 --- a/api/downloads.py +++ b/api/downloads.py @@ -121,7 +121,7 @@ def register_routes(bp): try: from utils.async_helpers import run_async - soulseek = current_app.soulsync.get("soulseek_client") + soulseek = current_app.soulsync.get("download_orchestrator") if not soulseek: return api_error("NOT_AVAILABLE", "Soulseek client not configured.", 503) @@ -138,7 +138,7 @@ def register_routes(bp): """Cancel all active downloads and clear completed ones.""" try: from utils.async_helpers import run_async - soulseek = current_app.soulsync.get("soulseek_client") + soulseek = current_app.soulsync.get("download_orchestrator") if not soulseek: return api_error("NOT_AVAILABLE", "Soulseek client not configured.", 503) diff --git a/api/request.py b/api/request.py index 820c6fc3..56c2d8ef 100644 --- a/api/request.py +++ b/api/request.py @@ -104,7 +104,7 @@ def _run_search_and_download(request_id, query, notify_url): if request_id in _pending_requests: _pending_requests[request_id]['status'] = 'searching' - soulseek = current_app._get_current_object().soulsync.get('soulseek_client') + soulseek = current_app._get_current_object().soulsync.get('download_orchestrator') if not soulseek: with _requests_lock: if request_id in _pending_requests: diff --git a/api/search.py b/api/search.py index 48a237c7..c2290fd3 100644 --- a/api/search.py +++ b/api/search.py @@ -2,10 +2,14 @@ Search endpoints — search external sources (Spotify, iTunes, Hydrabase). """ +import logging + from flask import request, current_app from .auth import require_api_key from .helpers import api_success, api_error +logger = logging.getLogger(__name__) + def register_routes(bp): @@ -38,8 +42,8 @@ def register_routes(bp): if hydra_results: tracks = [_serialize_track(t) for t in hydra_results] return api_success({"tracks": tracks, "source": "hydrabase"}) - except Exception: - pass + except Exception as e: + logger.debug("hydrabase search failed: %s", e) spotify = ctx.get("spotify_client") from core.metadata_service import get_primary_source, get_primary_client diff --git a/api/system.py b/api/system.py index 290da76e..c462d5f1 100644 --- a/api/system.py +++ b/api/system.py @@ -2,12 +2,15 @@ System endpoints — status, activity feed, stats. """ +import logging import time from flask import current_app from .auth import require_api_key from .helpers import api_success, api_error +logger = logging.getLogger(__name__) + def register_routes(bp): @@ -26,7 +29,7 @@ def register_routes(bp): spotify = ctx.get("spotify_client") spotify_ok = bool(spotify and spotify.is_authenticated()) - soulseek = ctx.get("soulseek_client") + soulseek = ctx.get("download_orchestrator") soulseek_ok = bool(soulseek) hydrabase = ctx.get("hydrabase_client") @@ -35,8 +38,8 @@ def register_routes(bp): try: ws, _ = hydrabase.get_ws_and_lock() hydrabase_ok = ws is not None and ws.connected - except Exception: - pass + except Exception as e: + logger.debug("hydrabase status probe failed: %s", e) return api_success({ "uptime": f"{hours}h {minutes}m {seconds}s", diff --git a/beatport_unified_scraper.py b/beatport_unified_scraper.py index 8202981c..950edded 100644 --- a/beatport_unified_scraper.py +++ b/beatport_unified_scraper.py @@ -2612,7 +2612,7 @@ class BeatportUnifiedScraper: result['title'] = title except Exception as e: - pass # Silently handle URL extraction errors + logger.debug("URL slug extraction: %s", e) return result @@ -3186,8 +3186,8 @@ class BeatportUnifiedScraper: except Exception: continue - except Exception: - pass + except Exception as e: + logger.debug("parse release tracks failed: %s", e) return tracks diff --git a/config/settings.py b/config/settings.py index 94105426..18c42124 100644 --- a/config/settings.py +++ b/config/settings.py @@ -495,6 +495,16 @@ class ConfigManager: "hifi_download": { "quality": "lossless", # Options: "low", "high", "lossless", "hires" }, + "hifi": { + "embed_tags": True, + "tags": { + "track_id": True, + "artist_id": True, + "isrc": True, + "bpm": True, + "copyright": True, + } + }, "lidarr_download": { "url": "", "api_key": "", @@ -502,6 +512,13 @@ class ConfigManager: "quality_profile": "Any", "cleanup_after_import": True, }, + "soundcloud_download": { + # Anonymous-only for now — SoundCloud Go+ OAuth tier could be + # added later, with credentials living under a "session" subkey + # alongside Tidal/Qobuz. No quality knob: anonymous SoundCloud + # caps at the upload's transcoding (typically 128 kbps MP3 or + # AAC). yt-dlp resolves bestaudio at download time. + }, "listenbrainz": { "base_url": "", "token": "", diff --git a/core/acoustid_verification.py b/core/acoustid_verification.py index a7cc3d0d..c00008ec 100644 --- a/core/acoustid_verification.py +++ b/core/acoustid_verification.py @@ -15,6 +15,7 @@ from typing import Optional, Dict, Any, Tuple, List from enum import Enum from utils.logging_config import get_logger from core.acoustid_client import AcoustIDClient +from core.matching_engine import MusicMatchingEngine from core.musicbrainz_client import MusicBrainzClient logger = get_logger("acoustid.verification") @@ -24,6 +25,25 @@ MIN_ACOUSTID_SCORE = 0.80 # Minimum AcoustID fingerprint score to trust TITLE_MATCH_THRESHOLD = 0.70 # Title similarity needed to consider a match ARTIST_MATCH_THRESHOLD = 0.60 # Artist similarity needed to consider a match +# Single matching-engine instance so version detection reuses the same patterns +# used by the pre-download Soulseek matcher (remix / live / acoustic / +# instrumental / etc). detect_version_type doesn't use self state, so one +# shared instance is fine. +_match_engine_for_version = MusicMatchingEngine() + + +def _detect_title_version(title: str) -> str: + """Return version label for a track title. + + Returns ``'original'`` when no version marker is detected, otherwise one + of the labels produced by ``MusicMatchingEngine.detect_version_type`` + (``'instrumental'``, ``'live'``, ``'acoustic'``, ``'remix'``, etc). + """ + if not title: + return 'original' + version_type, _ = _match_engine_for_version.detect_version_type(title) + return version_type + class VerificationResult(Enum): """Possible outcomes of audio verification.""" @@ -286,6 +306,33 @@ class AcoustIDVerification: f"(title_sim={title_sim:.2f}, artist_sim={artist_sim:.2f})" ) + # Step 4b: Version-mismatch gate. + # + # The ``_normalize`` step deliberately strips parentheticals and + # version tags ("(Instrumental)", "- Live", etc) so that legit + # name variations don't fail the title-similarity comparison. + # That same stripping made it impossible to tell a vocal track + # apart from its instrumental: "In My Feelings" and "In My + # Feelings (Instrumental)" both normalize to "in my feelings", + # the title sim ends up 1.0, and the file passes verification + # even though it's the wrong cut. + # + # Detect the version on each side BEFORE normalization runs. + # If the expected track and the AcoustID-matched recording + # disagree on version (one is original, the other is + # instrumental / live / remix / acoustic / etc), reject — the + # fingerprint identified a real song but it's not the one the + # caller asked for. + expected_version = _detect_title_version(expected_track_name) + matched_version = _detect_title_version(matched_title) + if expected_version != matched_version: + msg = ( + f"Version mismatch: expected '{expected_track_name}' ({expected_version}) " + f"but file is '{matched_title}' ({matched_version})" + ) + logger.warning(f"AcoustID verification FAILED (version mismatch) - {msg}") + return VerificationResult.FAIL, msg + # Step 5: Decide pass/fail based on similarity if title_sim >= TITLE_MATCH_THRESHOLD and artist_sim >= ARTIST_MATCH_THRESHOLD: msg = ( @@ -343,9 +390,15 @@ class AcoustIDVerification: # Title doesn't match — check ALL recordings for any title/artist match # (the best combined match might not be the right one if there are many results) + # Skip recordings whose version (instrumental/live/etc) disagrees with + # what the caller asked for — the version mismatch above checked + # only the best recording, but a wrong-version variant could still + # win this fallback scan if its bare title matched. for rec in recordings: t = rec.get('title') or '' a = rec.get('artist') or '' + if _detect_title_version(t) != expected_version: + continue if (_similarity(expected_track_name, t) >= TITLE_MATCH_THRESHOLD and _similarity(expected_artist_name, a) >= ARTIST_MATCH_THRESHOLD): msg = ( @@ -356,27 +409,61 @@ class AcoustIDVerification: return VerificationResult.PASS, msg # No match found — but if fingerprint score is very high (≥0.95) - # AND there's partial similarity in title or artist, the mismatch is - # likely a language/script difference (e.g. Japanese kanji vs English). - # Skip rather than quarantine a correct file. - # But if both title AND artist similarity are very low, the download - # source gave us a completely wrong file — fail it. - if best_score >= 0.95 and (title_sim >= 0.55 or artist_sim >= ARTIST_MATCH_THRESHOLD): - top = recordings[0] + # AND we have evidence the mismatch is a language/script case + # (rather than two genuinely different songs by the same artist), + # skip rather than quarantine a correct file. Two routes: + # + # (a) Either side of the comparison contains non-ASCII characters + # — strong signal of transliteration / kanji↔roman cases. + # Artist must still be a strong match to use this path. + # (b) Both title AND artist similarity are very high (the song + # is recognizably the same with minor punctuation / casing + # differences that fell below the strict match thresholds). + # + # The OLD logic was ``title_sim >= 0.55 OR artist_sim >= match``. + # That fired for English-vs-English songs by the same artist that + # share NO actual content — e.g. "R.O.T.C (Interlude)" by + # Kendrick Lamar getting accepted as "Rich (Interlude)" by + # Kendrick Lamar because the artist matched perfectly and + # "interlude" was shared in both titles. Reported by user when + # downloading Mr. Morale: three tracks (Rich Interlude, Savior + # Interlude, Savior) all received the wrong R.O.T.C audio file + # because of this leak. + top = recordings[0] + top_title = top.get('title', '?') or '' + top_artist = top.get('artist', '?') or '' + has_non_ascii = ( + any(ord(c) > 127 for c in (expected_track_name or '')) + or any(ord(c) > 127 for c in top_title) + ) + language_script_skip = ( + best_score >= 0.95 + and has_non_ascii + and artist_sim >= ARTIST_MATCH_THRESHOLD + ) + high_confidence_strong_match_skip = ( + best_score >= 0.95 + and title_sim >= 0.80 + and artist_sim >= ARTIST_MATCH_THRESHOLD + ) + if language_script_skip or high_confidence_strong_match_skip: + reason = ( + "likely same song in different language/script" + if language_script_skip + else "title/artist match within tolerance" + ) msg = ( f"Title/artist mismatch but fingerprint confidence very high ({best_score:.2f}): " - f"AcoustID='{top.get('title', '?')}' by '{top.get('artist', '?')}', " + f"AcoustID='{top_title}' by '{top_artist}', " f"expected '{expected_track_name}' by '{expected_artist_name}' — " - f"likely same song in different language/script" + f"{reason}" ) logger.info(f"AcoustID verification SKIPPED (high confidence) - {msg}") return VerificationResult.SKIP, msg # Low fingerprint score + no metadata match — file is likely wrong - top = recordings[0] - top_title = top.get('title', '?') - top_artist = top.get('artist', '?') - + # `top`, `top_title`, `top_artist` already resolved above for the + # skip-eligibility check. msg = ( f"Audio mismatch: file identified as '{top_title}' by '{top_artist}', " f"expected '{expected_track_name}' by '{expected_artist_name}' " diff --git a/core/album_consistency.py b/core/album_consistency.py index e913b8cc..e277d02c 100644 --- a/core/album_consistency.py +++ b/core/album_consistency.py @@ -195,8 +195,8 @@ def _find_best_release(album_name, artist_name, track_count, mb_service): sr_id = sr.get('id', '') if sr_id and sr_id not in candidate_mbids: candidate_mbids.append(sr_id) - except Exception: - pass + except Exception as e: + logger.debug("search_release fallback failed: %s", e) if not candidate_mbids: logger.info(f"No MB release found for '{album_name}' by '{artist_name}'") diff --git a/core/api_call_tracker.py b/core/api_call_tracker.py index f90a6dae..7fb561e1 100644 --- a/core/api_call_tracker.py +++ b/core/api_call_tracker.py @@ -302,8 +302,8 @@ class ApiCallTracker: try: if os.path.exists(_PERSIST_PATH + '.tmp'): os.remove(_PERSIST_PATH + '.tmp') - except Exception: - pass + except Exception as e: + logger.debug("remove stale tmp file failed: %s", e) def _load(self): """Restore 24h minute history from disk. Called on init.""" diff --git a/core/artists/liked_match.py b/core/artists/liked_match.py index 9607e31d..324d8cbc 100644 --- a/core/artists/liked_match.py +++ b/core/artists/liked_match.py @@ -89,20 +89,20 @@ def _match_liked_artists_to_all_sources(database, profile_id: int): search_clients['spotify'] = spotify_client try: search_clients['itunes'] = _get_itunes_client() - except Exception: - pass + except Exception as e: + logger.debug("itunes client init failed: %s", e) try: search_clients['deezer'] = _get_deezer_client() - except Exception: - pass + except Exception as e: + logger.debug("deezer client init failed: %s", e) try: dc = _get_discogs_client() # Only use Discogs if token is configured from config.settings import config_manager as _cm if _cm.get('discogs.token', ''): search_clients['discogs'] = dc - except Exception: - pass + except Exception as e: + logger.debug("discogs client init failed: %s", e) # Reuse watchlist scanner's fuzzy matching logic from core.watchlist_scanner import WatchlistScanner @@ -167,8 +167,8 @@ def _match_liked_artists_to_all_sources(database, profile_id: int): harvested_ids[col] = str(r[col]) if _valid_image(r.get('thumb_url')): best_image = r['thumb_url'] - except Exception: - pass + except Exception as e: + logger.debug("library artist lookup failed: %s", e) # 2. Watchlist artists try: @@ -186,8 +186,8 @@ def _match_liked_artists_to_all_sources(database, profile_id: int): harvested_ids[col] = str(wl[col]) if _valid_image(wl.get('image_url')) and not best_image: best_image = wl['image_url'] - except Exception: - pass + except Exception as e: + logger.debug("watchlist artist lookup failed: %s", e) # 3. Metadata cache (all sources) try: @@ -203,8 +203,8 @@ def _match_liked_artists_to_all_sources(database, profile_id: int): harvested_ids[col] = row['entity_id'] if _valid_image(row['image_url']) and not best_image: best_image = row['image_url'] - except Exception: - pass + except Exception as e: + logger.debug("metadata cache lookup failed: %s", e) # --- API STRATEGIES (search each missing source) --- # Same pattern as watchlist scanner's _backfill_missing_ids @@ -287,8 +287,8 @@ def _backfill_liked_artist_images(database, profile_id: int, search_clients: dic artist_data = sp.sp.artist(r['spotify_artist_id']) if artist_data and artist_data.get('images'): image_url = artist_data['images'][0]['url'] - except Exception: - pass + except Exception as e: + logger.debug("spotify artist image fetch failed: %s", e) # Try Deezer (direct image URL from ID) if not image_url and r.get('deezer_artist_id'): @@ -302,8 +302,8 @@ def _backfill_liked_artist_images(database, profile_id: int, search_clients: dic (image_url, r['id']) ) filled += 1 - except Exception: - pass + except Exception as e: + logger.debug("liked artist image update failed: %s", e) time.sleep(0.3) conn.commit() diff --git a/core/artists/map.py b/core/artists/map.py index 09e880ec..5e6f130a 100644 --- a/core/artists/map.py +++ b/core/artists/map.py @@ -158,8 +158,8 @@ def get_artist_map_data(): if r.get('genres'): try: genres = json.loads(r['genres']) - except Exception: - pass + except Exception as e: + logger.debug("similar node genres parse failed: %s", e) nodes.append({ 'id': idx, 'name': r['similar_artist_name'], @@ -232,8 +232,8 @@ def get_artist_map_data(): if cr['genres']: try: genres = json.loads(cr['genres']) if isinstance(cr['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("backfill cache genres parse failed: %s", e) cache_by_name[cn][source] = { 'id': cr['entity_id'], 'image_url': cr['image_url'] or '', @@ -280,8 +280,8 @@ def get_artist_map_data(): an = _norm(r['artist_name']) if an and an not in _album_art: _album_art[an] = r['image_url'] - except Exception: - pass + except Exception as e: + logger.debug("artist map album-art cache build failed: %s", e) for n in nodes: if not n.get('image_url') or not n['image_url'].startswith('http'): nn = _norm(n['name']) @@ -330,8 +330,8 @@ def get_artist_map_genre_list(): if g and isinstance(g, str): gl = g.lower().strip() genre_counts[gl] = genre_counts.get(gl, 0) + 1 - except Exception: - pass + except Exception as e: + logger.debug("genre count row parse failed: %s", e) # Sort by count descending sorted_genres = sorted(genre_counts.items(), key=lambda x: -x[1]) @@ -404,8 +404,8 @@ def get_artist_map_genres(): if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("cache artist genres parse failed: %s", e) src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} kwargs = {src_map.get(r['source'], 'spotify_id'): r['entity_id']} _add(r['name'], image_url=r['image_url'], genres=genres, source='cache', popularity=r['popularity'] or 0, **kwargs) @@ -421,8 +421,8 @@ def get_artist_map_genres(): if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("similar artist genres parse failed: %s", e) _add(r['similar_artist_name'], image_url=r['image_url'], genres=genres, spotify_id=r['similar_artist_spotify_id'], itunes_id=r['similar_artist_itunes_id'], deezer_id=r['similar_artist_deezer_id'], source='similar', popularity=r['popularity'] or 0) @@ -445,8 +445,8 @@ def get_artist_map_genres(): if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("library artist genres parse failed: %s", e) img = r['thumb_url'] if r['thumb_url'] and r['thumb_url'].startswith('http') else None _add(r['name'], image_url=img, genres=genres, source='library') @@ -550,8 +550,8 @@ def get_artist_map_genres(): nn = (r['artist_name'] or '').lower().strip() if nn and nn not in _album_art_cache: _album_art_cache[nn] = r['image_url'] - except Exception: - pass + except Exception as e: + logger.debug("genre map cache build failed: %s", e) for n in nodes: img = n.get('image_url', '') @@ -650,8 +650,8 @@ def get_artist_map_explore(): if row['genres']: try: center_genres = json.loads(row['genres']) if isinstance(row['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("initial center genres parse failed: %s", e) # Check watchlist + library if not in cache if not artist_found and not artist_id: @@ -722,8 +722,8 @@ def get_artist_map_explore(): if r['genres'] and not center_genres: try: center_genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("center genres parse failed: %s", e) # Add center node center_idx = 0 @@ -787,8 +787,8 @@ def get_artist_map_explore(): popularity=sa.get('popularity', 0), similar_artist_deezer_id=sa.get('deezer_id') ) - except Exception: - pass + except Exception as e: + logger.debug("similar artist insert failed: %s", e) # Re-query from DB to get consistent format if id_values: placeholders = ','.join(['?'] * len(id_values)) @@ -828,8 +828,8 @@ def get_artist_map_explore(): if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("ring1 genres parse failed: %s", e) img = r['image_url'] if r['image_url'] and r['image_url'].startswith('http') else '' nodes.append({ 'id': idx, 'name': r['similar_artist_name'], 'image_url': img, @@ -889,8 +889,8 @@ def get_artist_map_explore(): if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("ring2 genres parse failed: %s", e) img = r['image_url'] if r['image_url'] and r['image_url'].startswith('http') else '' nodes.append({ 'id': idx, 'name': r['similar_artist_name'], 'image_url': img, @@ -928,8 +928,8 @@ def get_artist_map_explore(): if not n['genres'] and cr['genres']: try: n['genres'] = json.loads(cr['genres'])[:5] if isinstance(cr['genres'], str) else [] - except Exception: - pass + except Exception as e: + logger.debug("explorer node genres parse failed: %s", e) # Harvest missing IDs from cache src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} k = src_map.get(cr['source']) @@ -951,8 +951,8 @@ def get_artist_map_explore(): n['image_url'] = artist_data['images'][0]['url'] if not n['genres'] and artist_data.get('genres'): n['genres'] = artist_data['genres'][:5] - except Exception: - pass + except Exception as e: + logger.debug("spotify artist image fallback failed: %s", e) # Album art fallback (iTunes artists have no artist images) if not n['image_url']: diff --git a/core/artists/quality.py b/core/artists/quality.py index 3bf6dec6..737d9339 100644 --- a/core/artists/quality.py +++ b/core/artists/quality.py @@ -2,8 +2,8 @@ `enhance_artist_quality(artist_id, track_ids, deps)` is the route-handler body for the `/api/library/artist//enhance` endpoint. It walks -the user's selected tracks, finds the best Spotify (preferred) or iTunes -(fallback) match for each, and queues high-quality re-downloads on the +the user's selected tracks, finds the best metadata match against the +configured primary source, and queues high-quality re-downloads on the wishlist with `source_type='enhance'`. Per-track flow: @@ -12,13 +12,43 @@ Per-track flow: front from `database.get_artist_full_detail`). 2. Read current quality tier from the file extension. 3. Build `matched_track_data` for the wishlist entry, in priority order: - - Direct Spotify lookup via stored `spotify_track_id` (preferred). - - Spotify search fallback using matching_engine queries. - - iTunes/fallback source search. -4. Add to wishlist via `wishlist_service.add_spotify_track_to_wishlist` + - **Direct lookup using stored source IDs** — for every source the + user has configured, if the library track has the corresponding + stored ID (`spotify_track_id` / `deezer_id` / `itunes_track_id` / + `soul_id`), call `client.get_track_details(stored_id)` and convert + the result to the wishlist payload. First success wins; the user's + configured primary source is tried first. Mirrors what Download + Discography does — stable IDs straight to the source's API, no + fuzzy text matching. + - **Multi-source parallel text search fallback** — if no stored ID + resolved, run the shared `core.metadata.multi_source_search` + against every configured source in parallel and pick the best + cross-source match (auto-accept threshold 0.7). +4. Validate the match has non-empty title, album, and artists. Reject + matches with empty fields — those propagated as + "unknown artist - unknown album - unknown track" wishlist entries + pre-fix because the wishlist payload normalizer's truthy-check + passthrough accepted dicts with empty string fields. +5. Add to wishlist via `wishlist_service.add_spotify_track_to_wishlist` with `source_type='enhance'` and a `source_context` carrying the original file path, format tier, bitrate, and artist name. -5. Tally `enhanced_count` / `failed_count` / per-track failure reasons. +6. Tally `enhanced_count` / `failed_count` / per-track failure reasons. + +The flow originally had Spotify-only logic with an iTunes search-only +fallback. Two failure modes drove the rewrite: + +- Users with neither Spotify nor Deezer connected got silent failures + ("unknown artist - unknown album - unknown track" wishlist entries) + because iTunes's text search returned junk matches with empty fields + that cleared the 0.7 confidence threshold. +- Library tracks with messy tags ("Title (Live)", featured artists in + the artist field, etc.) failed fuzzy text search even when a perfect + stored ID was available — Download Discography had no such problem + because it resolves albums by stable ID. + +Direct-lookup-via-stored-ID matches the Download Discography contract +for every source where we have an ID column. Text search is only the +fallback now. Returns `(payload_dict, http_status_code)` so the route wrapper can `jsonify()` and return. @@ -26,28 +56,286 @@ Returns `(payload_dict, http_status_code)` so the route wrapper can from __future__ import annotations -import logging import os from dataclasses import dataclass -from typing import Any, Callable +from typing import Any, Callable, Optional -logger = logging.getLogger(__name__) +from utils.logging_config import get_logger + +logger = get_logger('artists.quality') @dataclass class ArtistQualityDeps: """Bundle of cross-cutting deps the artist quality enhancement needs.""" - spotify_client: Any matching_engine: Any get_database: Callable[[], Any] get_wishlist_service: Callable[[], Any] get_current_profile_id: Callable[[], int] get_quality_tier_from_extension: Callable - get_metadata_fallback_client: Callable[[], Any] + # Returns ``[(source_name, client), ...]`` for every metadata source + # the user has configured. Powers both the direct-lookup fast path + # (resolves stored source IDs straight from each source's API, + # like Download Discography) and the multi-source parallel text + # search fallback (shared with Track Redownload via + # ``core.metadata.multi_source_search``). + get_metadata_search_sources: Callable[[], list] + + +def _has_complete_metadata(payload: Optional[dict]) -> bool: + """Reject matches with empty / missing core fields. Pre-fix, iTunes + returned matches that cleared the 0.7 confidence threshold while + having empty artist / album / title — those propagated as junk + wishlist entries displayed as 'unknown artist - unknown album - + unknown track'.""" + if not payload: + return False + if not (payload.get('name') or '').strip(): + return False + artists = payload.get('artists') or [] + has_artist = any( + (a.get('name') or '').strip() if isinstance(a, dict) else (a or '').strip() + for a in artists + ) + if not has_artist: + return False + album = payload.get('album') or {} + if isinstance(album, dict): + if not (album.get('name') or '').strip(): + return False + elif not (album or '').strip(): + return False + return True + + +def _build_payload_from_track(track_obj) -> dict: + """Build a Spotify-shaped wishlist payload from any metadata source's + Track-shaped object (Spotify Track, iTunes Track, Deezer Track, + Discogs Track — they all have the same .id / .name / .artists / + .album / .duration_ms / etc shape because each client mimics + Spotify's surface). + + The wishlist's downstream pipeline expects Spotify shape; this helper + is the single place that knows how to produce it. Replaces the + duplicated payload construction that used to live in the Spotify + search path AND the iTunes fallback path. + + Does NOT substitute defaults for missing artists / album / title — + ``_has_complete_metadata`` rejects empty matches downstream so the + user sees a clear failure instead of a junk wishlist entry with + fabricated values. + """ + image_url = getattr(track_obj, 'image_url', '') or '' + album_images = ( + [{'url': image_url, 'height': 600, 'width': 600}] + if image_url else [] + ) + artist_names = list(getattr(track_obj, 'artists', None) or []) + return { + 'id': getattr(track_obj, 'id', ''), + 'name': getattr(track_obj, 'name', '') or '', + 'artists': [{'name': a} for a in artist_names], + 'album': { + 'name': getattr(track_obj, 'album', '') or '', + 'artists': [{'name': a} for a in artist_names], + 'album_type': getattr(track_obj, 'album_type', None) or 'album', + 'images': album_images, + 'release_date': getattr(track_obj, 'release_date', '') or '', + 'total_tracks': 1, + }, + 'duration_ms': getattr(track_obj, 'duration_ms', 0) or 0, + 'track_number': getattr(track_obj, 'track_number', None) or 1, + 'disc_number': getattr(track_obj, 'disc_number', None) or 1, + 'popularity': getattr(track_obj, 'popularity', None) or 0, + 'preview_url': getattr(track_obj, 'preview_url', None), + 'external_urls': getattr(track_obj, 'external_urls', None) or {}, + } + + +# Map metadata source name → DB column on the ``tracks`` table that +# stores that source's native track ID. Used to drive the direct-lookup +# fast path: when a library track has a stored ID for source X and the +# user has source X configured, skip fuzzy text search and resolve +# straight from X's API. Mirrors what Download Discography does — stable +# IDs all the way, no fuzzy text matching. +# +# Discogs is release-based and has no per-track ID column; not listed +# here, so direct lookup never tries Discogs (search-fallback still +# runs for Discogs as one of the parallel sources). +_STORED_ID_COLUMNS = { + 'spotify': 'spotify_track_id', + 'deezer': 'deezer_id', + 'itunes': 'itunes_track_id', + 'hydrabase': 'soul_id', +} + + +def _enhanced_to_wishlist_payload(enhanced: dict, + fallback_title: str, + fallback_artist: str, + fallback_album: str) -> Optional[dict]: + """Convert a ``get_track_details`` enhanced-shape dict to the + Spotify-shape wishlist payload. + + Every metadata source's ``get_track_details`` returns the same + "enhanced" intermediate shape (top-level ``id``, ``name``, + ``artists`` as a list of strings, ``album.artists`` as strings), + documented and pinned across spotify_client / itunes_client / + deezer_client / hydrabase_client. The wishlist downstream expects + Spotify's native shape (``artists`` as ``[{'name': ...}]``), so + this helper does the conversion in one place. + + Spotify's ``raw_data`` field is already in wishlist shape (the + raw Spotify API response), so we return it as-is when detected, + preserving full ``album.images`` and ``external_urls`` that the + enhanced top-level fields drop. Other sources' ``raw_data`` is + in source-native shape and gets ignored. + """ + if not enhanced: + return None + raw = enhanced.get('raw_data') + if isinstance(raw, dict): + raw_artists = raw.get('artists') + if (isinstance(raw_artists, list) and raw_artists + and isinstance(raw_artists[0], dict)): + return raw + + artists = enhanced.get('artists') or [fallback_artist] + album_data = enhanced.get('album') or {} + album_artists = album_data.get('artists') or artists + + def _to_dict_artists(seq): + return [a if isinstance(a, dict) else {'name': a} for a in seq] + + image_url = enhanced.get('image_url') or '' + album_images_field = album_data.get('images') + if isinstance(album_images_field, list) and album_images_field: + album_images = album_images_field + elif image_url: + album_images = [{'url': image_url, 'height': 600, 'width': 600}] + else: + album_images = [] + + return { + 'id': str(enhanced.get('id', '')), + 'name': enhanced.get('name') or fallback_title, + 'artists': _to_dict_artists(artists), + 'album': { + 'id': str(album_data.get('id', '')), + 'name': album_data.get('name') or fallback_album, + 'album_type': album_data.get('album_type', 'album'), + 'release_date': album_data.get('release_date', ''), + 'total_tracks': album_data.get('total_tracks', 1), + 'artists': _to_dict_artists(album_artists), + 'images': album_images, + }, + 'duration_ms': enhanced.get('duration_ms', 0), + 'track_number': enhanced.get('track_number', 1), + 'disc_number': enhanced.get('disc_number', 1), + 'popularity': enhanced.get('popularity', 0), + 'preview_url': enhanced.get('preview_url'), + 'external_urls': enhanced.get('external_urls', {}), + } + + +def _try_direct_lookup_all_sources(track: dict, + sources: list, + preferred_source: Optional[str], + title: str, + artist_name: str, + album_title: str + ) -> tuple: + """Try direct ID-based lookup on every source where the library + track has a stored ID. Returns ``(payload, source_name)`` on first + success, or ``(None, None)`` if no source has a stored ID with a + successful lookup. + + Mirrors what Download Discography does — stable IDs straight to the + source's API, no fuzzy text matching. Avoids the failure mode where + library text tags don't match the source's canonical title (the + Discord report case: track tagged "Title (Live)" and source has + "Title" → fuzzy search misses, but stored ID resolves directly). + + Preferred source attempted first when present in ``sources``, + typically the user's configured primary metadata source — so a + Deezer-primary user gets Deezer art / album shape on the wishlist + entry instead of whichever source happened to have a stored ID + first in iteration order. + """ + def _priority(entry): + name = entry[0] + return 0 if name == preferred_source else 1 + ordered = sorted(sources, key=_priority) + + for source_name, client in ordered: + column = _STORED_ID_COLUMNS.get(source_name) + if not column: + continue + stored_id = track.get(column) + if not stored_id: + continue + if not hasattr(client, 'get_track_details'): + continue + try: + enhanced = client.get_track_details(str(stored_id)) + except Exception as exc: + logger.error( + f"[Enhance] {source_name} direct lookup failed for " + f"ID {stored_id}: {exc}" + ) + continue + if not enhanced: + continue + payload = _enhanced_to_wishlist_payload( + enhanced, title, artist_name, album_title, + ) + if _has_complete_metadata(payload): + logger.info( + f"[Enhance] Direct lookup matched: {source_name} " + f"ID {stored_id} → '{payload.get('name')}'" + ) + return payload, source_name + + return None, None + + +# Minimum match-score threshold for accepting a search-fallback match +# without user confirmation. Mirrors the legacy threshold the enhance +# flow has always used. +_AUTO_ACCEPT_SCORE_THRESHOLD = 0.7 def enhance_artist_quality(artist_id, track_ids, deps: ArtistQualityDeps): - """Add selected tracks to wishlist for quality enhancement re-download.""" + """Add selected tracks to wishlist for quality enhancement re-download. + + Per-track flow: + + 1. **Direct lookup using stored source IDs** (mirrors what Download + Discography does — stable IDs straight to the source's API, no + fuzzy text matching). For each source the user has configured, + if the library track has the corresponding stored ID + (``spotify_track_id`` / ``deezer_id`` / ``itunes_track_id`` / + ``soul_id``), call ``client.get_track_details(stored_id)`` and + convert to wishlist payload. First success wins; preferred + source (user's configured primary) tried first. + + 2. **Multi-source parallel text search fallback** (via the shared + ``core.metadata.multi_source_search`` module — same code path + Track Redownload uses) for tracks with no stored IDs / lookup + misses. + + 3. **Validation**: reject matches with empty title / album / artists + so the user sees a clear failure instead of an "unknown artist" + wishlist entry. + + Pre-refactor: only Spotify had a direct-lookup fast path; everything + else went through fuzzy text search. Discogs / Hydrabase / Deezer- + primary users got far worse coverage than Download Discography + despite both flows asking the same question. + """ + from core.metadata.multi_source_search import TrackQuery, search_all_sources + from core.metadata.registry import get_primary_source + try: if not track_ids: return {"success": False, "error": "No track IDs provided"}, 400 @@ -73,6 +361,18 @@ def enhance_artist_quality(artist_id, track_ids, deps: ArtistQualityDeps): track['_album_id'] = album.get('id') track_lookup[tid] = track + # Resolve every configured metadata source up front. + search_sources = deps.get_metadata_search_sources() + + # User's configured primary source — direct-lookup tries this + # first so Deezer-primary users get Deezer payloads on the + # wishlist entry (correct cover art / album shape) even when + # other sources also have stored IDs for the same track. + try: + preferred_source = get_primary_source() + except Exception: + preferred_source = None + enhanced_count = 0 failed_count = 0 failed_tracks = [] @@ -95,200 +395,67 @@ def enhance_artist_quality(artist_id, track_ids, deps: ArtistQualityDeps): title = track.get('title', '') or '' if not title.strip(): title = os.path.splitext(os.path.basename(file_path))[0] - spotify_tid = track.get('spotify_track_id') + album_title = track.get('_album_title', '') - # Build Spotify track data for wishlist matched_track_data = None + chosen_source = None - if spotify_tid and deps.spotify_client: - # Direct lookup via stored Spotify ID — raw_data has full Spotify API format + # 1. Direct lookup via every stored source ID — like Download + # Discography. Stable IDs, no fuzzy text matching. + if search_sources: + matched_track_data, chosen_source = _try_direct_lookup_all_sources( + track, search_sources, preferred_source, + title, artist_name, album_title, + ) + + # 2. Multi-source parallel text search fallback — for tracks + # with no stored IDs / lookup misses. + if not matched_track_data and search_sources: try: - track_details = deps.spotify_client.get_track_details(spotify_tid) - if track_details and track_details.get('raw_data'): - matched_track_data = track_details['raw_data'] - elif track_details: - # Enhanced format — rebuild with images for wishlist compatibility - album_data = track_details.get('album', {}) - album_images = [] - # Try to get album art from a full album lookup - if album_data.get('id'): - try: - full_album = deps.spotify_client.get_album(album_data['id']) - if full_album and full_album.get('images'): - album_images = full_album['images'] - except Exception: - pass - matched_track_data = { - 'id': spotify_tid, - 'name': track_details.get('name', title), - 'artists': [{'name': a} for a in track_details.get('artists', [artist_name])], - 'album': { - 'id': album_data.get('id', ''), - 'name': album_data.get('name', track.get('_album_title', '')), - 'album_type': album_data.get('album_type', 'album'), - 'release_date': album_data.get('release_date', ''), - 'total_tracks': album_data.get('total_tracks', 1), - 'artists': [{'name': a} for a in album_data.get('artists', [artist_name])], - 'images': album_images, - }, - 'duration_ms': track_details.get('duration_ms', track.get('duration', 0)), - 'track_number': track_details.get('track_number', track.get('track_number', 1)), - 'disc_number': track_details.get('disc_number', 1), - 'popularity': 0, - 'preview_url': None, - 'external_urls': {}, - } - except Exception as e: - logger.error(f"[Enhance] Spotify lookup failed for {spotify_tid}: {e}") - - if not matched_track_data and deps.spotify_client: - # Fallback: Spotify search matching — need full track data for wishlist - try: - temp_track = type('TempTrack', (), { - 'name': title, 'artists': [artist_name], - 'album': track.get('_album_title', '') - })() - search_queries = deps.matching_engine.generate_download_queries(temp_track) - best_match = None - best_match_raw = None - best_confidence = 0.0 - - for search_query in search_queries[:3]: # Limit queries - try: - results = deps.spotify_client.search_tracks(search_query, limit=5) - if not results: - continue - for sp_track in results: - artist_conf = max( - (deps.matching_engine.similarity_score( - deps.matching_engine.normalize_string(artist_name), - deps.matching_engine.normalize_string(a) - ) for a in (sp_track.artists or [artist_name])), - default=0 - ) - title_conf = deps.matching_engine.similarity_score( - deps.matching_engine.normalize_string(title), - deps.matching_engine.normalize_string(sp_track.name) - ) - combined = artist_conf * 0.5 + title_conf * 0.5 - # Small bonus for album tracks over singles - _at = getattr(sp_track, 'album_type', None) or '' - if _at == 'album': - combined += 0.02 - elif _at == 'ep': - combined += 0.01 - if combined > best_confidence and combined >= 0.7: - best_confidence = combined - best_match = sp_track - if best_confidence >= 0.9: - break - except Exception: - continue - - if best_match: - # Fetch full track data from Spotify for proper wishlist format - try: - full_details = deps.spotify_client.get_track_details(best_match.id) - if full_details and full_details.get('raw_data'): - matched_track_data = full_details['raw_data'] - else: - raise ValueError("No raw_data from get_track_details") - except Exception: - # Build from Track dataclass with image - album_images = [{'url': best_match.image_url}] if best_match.image_url else [] - matched_track_data = { - 'id': best_match.id, - 'name': best_match.name, - 'artists': [{'name': a} for a in best_match.artists], - 'album': { - 'name': best_match.album, - 'artists': [{'name': a} for a in best_match.artists], - 'album_type': 'album', - 'release_date': getattr(best_match, 'release_date', '') or '', - 'images': album_images, - }, - 'duration_ms': best_match.duration_ms, - 'popularity': best_match.popularity or 0, - 'preview_url': best_match.preview_url, - 'external_urls': best_match.external_urls or {}, - } - except Exception as e: - logger.error(f"[Enhance] Search match failed for {title}: {e}") - - # Fallback source when Spotify unavailable or no match found - if not matched_track_data: - try: - fallback_client = deps.get_metadata_fallback_client() - itunes_best = None - itunes_best_conf = 0.0 - - itunes_queries = deps.matching_engine.generate_download_queries( - type('TempTrack', (), { - 'name': title, 'artists': [artist_name], - 'album': track.get('_album_title', '') - })() + track_query = TrackQuery( + title=title, + artist=artist_name, + album=album_title, + duration_ms=track.get('duration', 0) or 0, + spotify_track_id=track.get('spotify_track_id'), + deezer_id=track.get('deezer_id'), ) + multi_result = search_all_sources(track_query, search_sources) + if multi_result.best_match and multi_result.best_match['score'] >= _AUTO_ACCEPT_SCORE_THRESHOLD: + chosen_source = multi_result.best_match['source'] + best_track_obj = multi_result.best_track() + if best_track_obj: + matched_track_data = _build_payload_from_track(best_track_obj) + except Exception as exc: + logger.error(f"[Enhance] Multi-source search failed for {title}: {exc}") - for search_query in itunes_queries[:3]: - try: - itunes_results = fallback_client.search_tracks(search_query, limit=5) - if not itunes_results: - continue - for it_track in itunes_results: - artist_conf = max( - (deps.matching_engine.similarity_score( - deps.matching_engine.normalize_string(artist_name), - deps.matching_engine.normalize_string(a) - ) for a in (it_track.artists or [artist_name])), - default=0 - ) - title_conf = deps.matching_engine.similarity_score( - deps.matching_engine.normalize_string(title), - deps.matching_engine.normalize_string(it_track.name) - ) - combined = artist_conf * 0.5 + title_conf * 0.5 - # Small bonus for album tracks over singles - _at = getattr(it_track, 'album_type', None) or '' - if _at == 'album': - combined += 0.02 - elif _at == 'ep': - combined += 0.01 - if combined > itunes_best_conf and combined >= 0.7: - itunes_best_conf = combined - itunes_best = it_track - if itunes_best_conf >= 0.9: - break - except Exception: - continue - - if itunes_best: - album_images = [{'url': itunes_best.image_url, 'height': 600, 'width': 600}] if itunes_best.image_url else [] - matched_track_data = { - 'id': itunes_best.id, - 'name': itunes_best.name, - 'artists': [{'name': a} for a in itunes_best.artists], - 'album': { - 'name': itunes_best.album, - 'artists': [{'name': a} for a in itunes_best.artists], - 'album_type': 'album', - 'images': album_images, - 'release_date': itunes_best.release_date or '', - 'total_tracks': 1, - }, - 'duration_ms': itunes_best.duration_ms, - 'track_number': itunes_best.track_number or 1, - 'disc_number': itunes_best.disc_number or 1, - 'popularity': itunes_best.popularity or 0, - 'preview_url': itunes_best.preview_url, - 'external_urls': itunes_best.external_urls or {}, - } - logger.warning(f"[Enhance] Fallback match for {title}: {itunes_best.artists[0]} - {itunes_best.name} (conf: {itunes_best_conf:.3f})") - except Exception as e: - logger.error(f"[Enhance] Fallback source failed for {title}: {e}") + # 3. Reject matches with empty / missing core fields. + if not _has_complete_metadata(matched_track_data): + if matched_track_data: + logger.warning( + f"[Enhance] {chosen_source} match for '{title}' rejected — " + f"empty title / album / artists (would render as 'unknown')" + ) + matched_track_data = None if not matched_track_data: failed_count += 1 - failed_tracks.append({'track_id': track_id, 'title': title, 'reason': 'No Spotify or fallback match'}) + source_list = ', '.join(name for name, _ in (search_sources or [])) + if not source_list: + reason = ( + 'No metadata source configured — connect Spotify / ' + 'iTunes / Deezer / Discogs / Hydrabase to enable enhance' + ) + else: + reason = ( + f'No usable match across {source_list} — ' + f'try connecting an additional metadata source' + ) + failed_tracks.append({ + 'track_id': track_id, + 'title': title, + 'reason': reason, + }) continue # Add to wishlist with enhance source diff --git a/core/audiodb_worker.py b/core/audiodb_worker.py index cfb323bc..37c404d9 100644 --- a/core/audiodb_worker.py +++ b/core/audiodb_worker.py @@ -140,8 +140,8 @@ class AudioDBWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null id table resolve failed: %s", e) continue diff --git a/core/auto_import_worker.py b/core/auto_import_worker.py index a564d862..be80a7d2 100644 --- a/core/auto_import_worker.py +++ b/core/auto_import_worker.py @@ -36,6 +36,11 @@ class FolderCandidate: disc_structure: Dict[int, List[str]] = field(default_factory=dict) # disc_num -> files folder_hash: str = '' is_single: bool = False # True for loose files in staging root + # True when the candidate "folder" is the staging root itself (user dropped + # disc folders directly into staging without an album wrapper). The name is + # meaningless ("Staging", "Music", etc.) — folder-name identification must + # be skipped or it will false-match against random albums. + is_staging_root: bool = False def _compute_folder_hash(audio_files: List[str]) -> str: @@ -58,7 +63,11 @@ def _read_file_tags(file_path: str) -> Dict[str, Any]: if audio and audio.tags: tags = audio.tags result['title'] = (tags.get('title', [''])[0] or '').strip() - result['artist'] = (tags.get('artist', [''])[0] or tags.get('albumartist', [''])[0] or '').strip() + # Prefer albumartist for album-level identification (per-track artist + # often includes features like "Kendrick Lamar, Drake" which fragment + # consensus when grouping tracks into an album). Fall back to artist + # for files that lack albumartist. + result['artist'] = (tags.get('albumartist', [''])[0] or tags.get('artist', [''])[0] or '').strip() result['album'] = (tags.get('album', [''])[0] or '').strip() # Date/year — try 'date' first, fall back to 'year' date_str = (tags.get('date', [''])[0] or tags.get('year', [''])[0] or '').strip() @@ -137,7 +146,15 @@ class AutoImportWorker: self._folder_snapshots: Dict[str, float] = {} # path -> mtime_sum self._processing_paths: set = set() # Paths currently being processed (skip on rescan) self._current_folder = '' - self._current_status = 'idle' + self._current_status = 'idle' # 'idle' | 'scanning' | 'processing' + # Live per-track progress so the UI can show "Processing Speak Now + # (3/14: Mine)" while a multi-track album is being post-processed. + # Without this, auto-import goes silent for the entire processing + # window (which can be 5+ minutes for a full album) since + # ``_record_result`` only fires after every track is done. + self._current_track_index = 0 + self._current_track_total = 0 + self._current_track_name = '' self._stats = {'scanned': 0, 'auto_processed': 0, 'pending_review': 0, 'failed': 0} self._last_scan_time = None @@ -173,6 +190,9 @@ class AutoImportWorker: 'paused': self.paused, 'current_folder': self._current_folder, 'current_status': self._current_status, + 'current_track_index': self._current_track_index, + 'current_track_total': self._current_track_total, + 'current_track_name': self._current_track_name, 'stats': self._stats.copy(), 'last_scan_time': self._last_scan_time, } @@ -287,11 +307,19 @@ class AutoImportWorker: has_strong_individual_matches = len(high_conf_matches) > 0 if (confidence >= threshold or has_strong_individual_matches) and auto_process: - # Phase 5: Auto-process — process all tracks that matched + # Phase 5: Auto-process — insert an in-progress row + # so the UI sees the import the moment it starts, + # then update it with the final status when done. effective_conf = max(confidence, min(m['confidence'] for m in high_conf_matches) if high_conf_matches else 0) logger.info(f"[Auto-Import] Processing {candidate.name} — " f"overall: {confidence:.0%}, {len(high_conf_matches)} strong matches, " f"{match_result.get('matched_count', 0)}/{match_result.get('total_tracks', '?')} tracks") + + in_progress_row_id = self._record_in_progress( + candidate, identification, match_result, + ) + self._current_status = 'processing' + success = self._process_matches(candidate, identification, match_result) status = 'completed' if success else 'failed' confidence = max(confidence, effective_conf) @@ -299,22 +327,38 @@ class AutoImportWorker: self._stats['auto_processed'] += 1 else: self._stats['failed'] += 1 + + # Reset live progress state regardless of outcome + self._current_track_index = 0 + self._current_track_total = 0 + self._current_track_name = '' + self._current_status = 'scanning' if not self.should_stop else 'idle' + + # Update the in-progress row in place — UI shows the + # final result without a separate insert race. + self._finalize_result(in_progress_row_id, status, confidence) elif confidence >= 0.7: status = 'pending_review' self._stats['pending_review'] += 1 logger.info(f"[Auto-Import] Medium confidence ({confidence:.0%}) — pending review: {candidate.name}") + self._record_result(candidate, status, confidence, + album_id=identification.get('album_id'), + album_name=identification.get('album_name'), + artist_name=identification.get('artist_name'), + image_url=identification.get('image_url'), + identification_method=identification.get('method'), + match_data=match_result) else: status = 'needs_identification' self._stats['failed'] += 1 logger.info(f"[Auto-Import] Low confidence ({confidence:.0%}) — needs manual ID: {candidate.name}") - - self._record_result(candidate, status, confidence, - album_id=identification.get('album_id'), - album_name=identification.get('album_name'), - artist_name=identification.get('artist_name'), - image_url=identification.get('image_url'), - identification_method=identification.get('method'), - match_data=match_result) + self._record_result(candidate, status, confidence, + album_id=identification.get('album_id'), + album_name=identification.get('album_name'), + artist_name=identification.get('artist_name'), + image_url=identification.get('image_url'), + identification_method=identification.get('method'), + match_data=match_result) except Exception as e: logger.error(f"[Auto-Import] Error processing {candidate.name}: {e}") @@ -322,6 +366,12 @@ class AutoImportWorker: self._stats['failed'] += 1 finally: self._processing_paths.discard(candidate.path) + # Defensive: if the inner code path didn't reset live + # progress (early raise, etc.), clear it so the UI + # doesn't show stale "processing track 3/14" forever. + self._current_track_index = 0 + self._current_track_total = 0 + self._current_track_name = '' # Rate limit between folders if self._interruptible_sleep(2): @@ -344,10 +394,10 @@ class AutoImportWorker: def _enumerate_folders(self, staging: str) -> List[FolderCandidate]: """Find album folder and single file candidates in staging directory (recursive).""" candidates = [] - self._scan_directory(staging, candidates) + self._scan_directory(staging, candidates, staging_root=staging) return candidates - def _scan_directory(self, directory: str, candidates: List[FolderCandidate]): + def _scan_directory(self, directory: str, candidates: List[FolderCandidate], staging_root: str = ''): """Recursively scan a directory for album folders and loose audio files.""" try: entries = sorted(os.listdir(directory)) @@ -403,12 +453,45 @@ class AutoImportWorker: disc_structure=disc_structure, folder_hash=folder_hash )) else: - # No audio files here — recurse into subdirectories - for sub_name, sub_path in subdirs: - # Skip disc folders at this level (they'll be handled by the parent album) - if DISC_FOLDER_RE.match(sub_name): - continue - self._scan_directory(sub_path, candidates) + # No loose audio files. If the only subdirs are disc folders, + # treat THIS directory as the album candidate (multi-disc album + # with no album-level loose files — common when a user drops + # `Album/Disc 1/`, `Album/Disc 2/` straight into staging, or + # drops `Disc 1/`, `Disc 2/` with the staging dir itself as + # the album root). + disc_subdirs = [(n, p) for n, p in subdirs if DISC_FOLDER_RE.match(n)] + non_disc_subdirs = [(n, p) for n, p in subdirs if not DISC_FOLDER_RE.match(n)] + + if disc_subdirs and not non_disc_subdirs: + disc_structure = {} + audio_files = [] + for sub_name, sub_path in disc_subdirs: + disc_num = int(DISC_FOLDER_RE.match(sub_name).group(1)) + try: + disc_files = [os.path.join(sub_path, f) for f in sorted(os.listdir(sub_path)) + if os.path.isfile(os.path.join(sub_path, f)) + and os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] + except OSError: + disc_files = [] + if disc_files: + disc_structure[disc_num] = disc_files + audio_files.extend(disc_files) + + if audio_files: + folder_name = os.path.basename(directory) + folder_hash = _compute_folder_hash(audio_files) + is_staging_root = bool(staging_root) and os.path.normpath(directory) == os.path.normpath(staging_root) + candidates.append(FolderCandidate( + path=directory, name=folder_name, audio_files=audio_files, + disc_structure=disc_structure, folder_hash=folder_hash, + is_staging_root=is_staging_root, + )) + return + + # Otherwise recurse into non-disc subdirs (disc folders only + # ever attach to a parent album, never stand alone). + for _sub_name, sub_path in non_disc_subdirs: + self._scan_directory(sub_path, candidates, staging_root=staging_root) def _is_folder_stable(self, candidate: FolderCandidate) -> bool: """Check if folder contents have stopped changing.""" @@ -450,10 +533,15 @@ class AutoImportWorker: if tag_result: return tag_result - # Strategy 2: Parse folder name - folder_result = self._identify_from_folder_name(candidate) - if folder_result: - return folder_result + # Strategy 2: Parse folder name (skip when the candidate is the staging + # root itself — the folder name is meaningless and will false-match + # against random albums in the metadata source). + if candidate.is_staging_root: + logger.info(f"[Auto-Import] Skipping folder-name identification for staging root '{candidate.name}' — would false-match. Falling through to AcoustID.") + else: + folder_result = self._identify_from_folder_name(candidate) + if folder_result: + return folder_result # Strategy 3: AcoustID fingerprint acoustid_result = self._identify_from_acoustid(candidate) @@ -508,8 +596,8 @@ class AutoImportWorker: # Keep weak AcoustID result as fallback if fp_result2 and (not result or fp_result2.get('identification_confidence', 0) > result.get('identification_confidence', 0)): result = fp_result2 - except Exception: - pass + except Exception as e: + logger.debug("acoustid fingerprint fallback failed: %s", e) # If we have good tag data (artist + title), prefer tag-based identification # over a weak metadata/AcoustID result — tags from post-processed files are reliable @@ -643,29 +731,47 @@ class AutoImportWorker: def _identify_from_tags(self, candidate: FolderCandidate) -> Optional[Dict]: """Try to identify album from embedded file tags.""" tags_list = [] - for f in candidate.audio_files[:20]: # Cap at 20 files + sampled = candidate.audio_files[:20] # Cap at 20 files + for f in sampled: tags = _read_file_tags(f) if tags['album'] and tags['artist']: tags_list.append(tags) - if len(tags_list) < max(1, len(candidate.audio_files) * 0.5): + if len(tags_list) < max(1, len(sampled) * 0.5): + logger.info(f"[Auto-Import] Tag identification rejected for '{candidate.name}' — only {len(tags_list)}/{len(sampled)} files have album+artist tags (need >=50%)") return None # Less than 50% of files have usable tags - # Check consistency — most common album+artist - album_artist_counts = {} + # Group by album first (album-level identity). Per-track artist often + # varies due to features ("Artist", "Artist, Drake", etc.) so grouping + # by (album, artist) fragments consensus on a real album. Pick the + # dominant album, then within that album pick the most-common artist + # (which will usually be the album's primary artist). + album_counts = {} for t in tags_list: - key = (t['album'].lower().strip(), t['artist'].lower().strip()) - album_artist_counts[key] = album_artist_counts.get(key, 0) + 1 + album_key = t['album'].lower().strip() + album_counts[album_key] = album_counts.get(album_key, 0) + 1 - if not album_artist_counts: + if not album_counts: return None - best_key, best_count = max(album_artist_counts.items(), key=lambda x: x[1]) - if best_count < len(tags_list) * 0.6: - return None # Tags too inconsistent + best_album, best_album_count = max(album_counts.items(), key=lambda x: x[1]) + if best_album_count < len(tags_list) * 0.6: + sample = ', '.join([f"'{a}' x{c}" for a, c in sorted(album_counts.items(), key=lambda x: -x[1])[:3]]) + logger.info(f"[Auto-Import] Tag identification rejected for '{candidate.name}' — best album '{best_album}' only {best_album_count}/{len(tags_list)} files (need >=60%). Top albums: {sample}") + return None - album_name, artist_name = best_key - return self._search_metadata_source(artist_name, album_name, 'tags', candidate) + # Most-common artist among files matching the dominant album + artist_counts = {} + for t in tags_list: + if t['album'].lower().strip() == best_album: + a = t['artist'].lower().strip() + if a: + artist_counts[a] = artist_counts.get(a, 0) + 1 + if not artist_counts: + return None + artist_name, _ = max(artist_counts.items(), key=lambda x: x[1]) + + return self._search_metadata_source(artist_name, best_album, 'tags', candidate) def _identify_from_folder_name(self, candidate: FolderCandidate) -> Optional[Dict]: """Try to identify album from folder name.""" @@ -994,8 +1100,8 @@ class AutoImportWorker: if folder_artist and folder_artist.lower() != artist_name.lower(): logger.info(f"[Auto-Import] Parent folder artist '{folder_artist}' differs from tag artist '{artist_name}' — using folder artist") artist_name = folder_artist - except Exception: - pass + except Exception as e: + logger.debug("folder artist override failed: %s", e) release_date = identification.get('release_date', '') or album_data.get('release_date', '') # Compute total discs @@ -1005,21 +1111,31 @@ class AutoImportWorker: processed = 0 errors = [] + all_matches = list(match_result.get('matches', [])) + # Surface track total for the UI's live-progress widget. Matches + # the loop denominator so users see "3/14" while it's working. + self._current_track_total = len(all_matches) - for match in match_result.get('matches', []): + for index, match in enumerate(all_matches, start=1): track = match['track'] file_path = match['file'] + track_name = track.get('name', 'Unknown') + track_number = track.get('track_number', 1) + disc_number = track.get('disc_number', 1) + track_id = track.get('id', '') + + # Update live progress BEFORE the per-track work so the UI + # sees the right "now processing track N: " the + # moment polling fires (every 5s). + self._current_track_index = index + self._current_track_name = track_name + if not os.path.exists(file_path): errors.append(f"File not found: {os.path.basename(file_path)}") continue try: - track_name = track.get('name', 'Unknown') - track_number = track.get('track_number', 1) - disc_number = track.get('disc_number', 1) - track_id = track.get('id', '') - # Build context matching the manual import format context_key = f"auto_import_{candidate.folder_hash}_{track_number}" context = { @@ -1086,34 +1202,107 @@ class AutoImportWorker: 'completed_tracks': str(processed), 'failed_tracks': str(len(errors)), }) - except Exception: - pass + except Exception as e: + logger.debug("automation emit failed: %s", e) return processed > 0 # ── Database ── + def _record_in_progress(self, candidate: FolderCandidate, identification: Dict, + match_result: Dict) -> Optional[int]: + """Insert a status='processing' row up-front so the UI can see + an in-flight import while it's still running. Returns the row's + id so ``_finalize_result`` can update the same row when done. + + Without this, auto-import goes silent for the entire processing + window (5+ minutes for a full album) — the existing + ``_record_result`` only fires after every track is post- + processed, so the UI sees nothing in history while the user + waits. + """ + try: + match_json = self._serialize_match_data(match_result) + conn = self.database._get_connection() + cursor = conn.cursor() + cursor.execute(""" + INSERT INTO auto_import_history + (folder_name, folder_path, folder_hash, status, confidence, album_id, album_name, + artist_name, image_url, total_files, matched_files, match_data, + identification_method, error_message, processed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + candidate.name, candidate.path, candidate.folder_hash, + 'processing', match_result.get('confidence', 0.0), + identification.get('album_id'), identification.get('album_name'), + identification.get('artist_name'), identification.get('image_url'), + len(candidate.audio_files), + match_result.get('matched_count', 0), + match_json, identification.get('method'), None, None, + )) + row_id = cursor.lastrowid + conn.commit() + conn.close() + return row_id + except Exception as e: + logger.error(f"Error recording in-progress auto-import row: {e}") + return None + + def _finalize_result(self, row_id: int, status: str, confidence: float, + error_message: Optional[str] = None) -> None: + """Update the in-progress row created by ``_record_in_progress`` + with the final outcome. Idempotent — safe to call even if the + row creation failed (row_id is None).""" + if not row_id: + return + try: + conn = self.database._get_connection() + cursor = conn.cursor() + cursor.execute(""" + UPDATE auto_import_history + SET status = ?, confidence = ?, error_message = ?, processed_at = ? + WHERE id = ? + """, ( + status, confidence, error_message, + datetime.now().isoformat() if status == 'completed' else None, + row_id, + )) + conn.commit() + conn.close() + except Exception as e: + logger.error(f"Error finalizing auto-import row {row_id}: {e}") + + def _serialize_match_data(self, match_data: Optional[Dict]) -> Optional[str]: + """Serialize match_result for storage. Strips the non-JSON-safe + ``album_data`` reference and per-match track dicts down to just + the fields the review UI uses.""" + if not match_data: + return None + try: + serializable = { + 'matches': [{'track_name': m['track']['name'], + 'track_number': m['track'].get('track_number', 0), + 'file': os.path.basename(m['file']), + 'confidence': m['confidence']} for m in match_data.get('matches', [])], + 'unmatched_files': [os.path.basename(f) for f in match_data.get('unmatched_files', [])], + 'total_tracks': match_data.get('total_tracks', 0), + 'matched_count': match_data.get('matched_count', 0), + 'coverage': match_data.get('coverage', 0), + } + return json.dumps(serializable) + except Exception: + return None + def _record_result(self, candidate: FolderCandidate, status: str, confidence: float, album_id: str = None, album_name: str = None, artist_name: str = None, image_url: str = None, identification_method: str = None, match_data: Dict = None, error_message: str = None): - """Record auto-import result to database.""" + """Record auto-import result to database (one-shot, no in-progress + upsert). Used for early-failure paths that never enter the + per-track processing loop (identification failures, match + failures, low-confidence skips).""" try: - # Serialize match data (strip non-serializable album_data) - match_json = None - if match_data: - serializable = { - 'matches': [{'track_name': m['track']['name'], - 'track_number': m['track'].get('track_number', 0), - 'file': os.path.basename(m['file']), - 'confidence': m['confidence']} for m in match_data.get('matches', [])], - 'unmatched_files': [os.path.basename(f) for f in match_data.get('unmatched_files', [])], - 'total_tracks': match_data.get('total_tracks', 0), - 'matched_count': match_data.get('matched_count', 0), - 'coverage': match_data.get('coverage', 0), - } - match_json = json.dumps(serializable) - + match_json = self._serialize_match_data(match_data) conn = self.database._get_connection() cursor = conn.cursor() cursor.execute(""" diff --git a/core/automation/progress.py b/core/automation/progress.py index 8f3e8f50..9306f19f 100644 --- a/core/automation/progress.py +++ b/core/automation/progress.py @@ -77,8 +77,8 @@ def update_progress( if socketio_emit is not None: try: socketio_emit('automation:progress', {str(automation_id): dict(state)}) - except Exception: - pass + except Exception as e: + logger.debug("socketio progress emit: %s", e) def get_running_progress() -> dict[str, dict]: @@ -121,8 +121,8 @@ def record_history( t0 = datetime.fromisoformat(started_at) t1 = datetime.fromisoformat(finished_at) duration = (t1 - t0).total_seconds() - except Exception: - pass + except Exception as e: + logger.debug("duration parse: %s", e) r_status = result.get('status', 'completed') if result else 'completed' if r_status == 'error': diff --git a/core/automation/signals.py b/core/automation/signals.py index a9ce9214..9b82c5a1 100644 --- a/core/automation/signals.py +++ b/core/automation/signals.py @@ -8,6 +8,9 @@ names from the saved automation set so the builder UI can autocomplete. from __future__ import annotations import json +import logging + +logger = logging.getLogger(__name__) def collect_known_signals(database) -> list[str]: @@ -38,6 +41,6 @@ def collect_known_signals(database) -> list[str]: signals.add(sig) except (json.JSONDecodeError, TypeError): pass - except Exception: - pass + except Exception as e: + logger.debug("collect known signals failed: %s", e) return sorted(signals) diff --git a/core/automation_engine.py b/core/automation_engine.py index a1789af4..d19cbb58 100644 --- a/core/automation_engine.py +++ b/core/automation_engine.py @@ -455,8 +455,10 @@ class AutomationEngine: if delay_minutes and delay_minutes > 0: # Initialize progress BEFORE delay so card glows during wait if self._progress_init_fn: - try: self._progress_init_fn(automation_id, auto.get('name', ''), action_type) - except Exception: pass + try: + self._progress_init_fn(automation_id, auto.get('name', ''), action_type) + except Exception as e: + logger.debug("event progress init (delay): %s", e) _delay_already_inited = True delay_seconds = int(delay_minutes) * 60 @@ -486,13 +488,17 @@ class AutomationEngine: logger.info(f"Event automation '{auto.get('name')}' skipped — {action_type} busy") # If progress was initialized during delay, finalize it if _delay_already_inited and self._progress_finish_fn: - try: self._progress_finish_fn(automation_id, result) - except Exception: pass + try: + self._progress_finish_fn(automation_id, result) + except Exception as e: + logger.debug("event progress finish (skipped): %s", e) else: # Initialize progress tracking (skip if already done during delay) if not _delay_already_inited and self._progress_init_fn: - try: self._progress_init_fn(automation_id, auto.get('name', ''), action_type) - except Exception: pass + try: + self._progress_init_fn(automation_id, auto.get('name', ''), action_type) + except Exception as e: + logger.debug("event progress init: %s", e) try: result = handler_info['handler'](action_config) or {} logger.info(f"Event automation '{auto.get('name')}' executed: {result.get('status', 'ok')}") @@ -501,8 +507,10 @@ class AutomationEngine: logger.error(f"Event automation '{auto.get('name')}' action failed: {e}") # Finalize progress tracking if self._progress_finish_fn: - try: self._progress_finish_fn(automation_id, result) - except Exception: pass + try: + self._progress_finish_fn(automation_id, result) + except Exception as e: + logger.debug("event progress finish: %s", e) # Merge event data into result for then-action variables merged = {**event_data, **result} @@ -531,8 +539,8 @@ class AutomationEngine: if self._history_record_fn: try: self._history_record_fn(automation_id, result) - except Exception: - pass + except Exception as e: + logger.debug("history record failed: %s", e) # --- Schedule Execution (timer-based) --- @@ -580,8 +588,10 @@ class AutomationEngine: if not skip_delay and delay_minutes and delay_minutes > 0: # Initialize progress BEFORE delay so card glows during wait if self._progress_init_fn: - try: self._progress_init_fn(automation_id, auto.get('name', ''), action_type) - except Exception: pass + try: + self._progress_init_fn(automation_id, auto.get('name', ''), action_type) + except Exception as e: + logger.debug("scheduled progress init (delay): %s", e) _delay_already_inited = True delay_seconds = int(delay_minutes) * 60 @@ -603,15 +613,19 @@ class AutomationEngine: logger.info(f"Automation '{auto['name']}' skipped — {action_type} already running") # If progress was initialized during delay, finalize it if _delay_already_inited and self._progress_finish_fn: - try: self._progress_finish_fn(automation_id, result) - except Exception: pass + try: + self._progress_finish_fn(automation_id, result) + except Exception as e: + logger.debug("scheduled progress finish (skipped): %s", e) self._finish_run(auto, automation_id, result, error=None) return # Initialize progress tracking (skip if already done during delay) if not _delay_already_inited and self._progress_init_fn: - try: self._progress_init_fn(automation_id, auto.get('name', ''), action_type) - except Exception: pass + try: + self._progress_init_fn(automation_id, auto.get('name', ''), action_type) + except Exception as e: + logger.debug("scheduled progress init: %s", e) # Execute the action error = None @@ -637,8 +651,10 @@ class AutomationEngine: # Finalize progress tracking if self._progress_finish_fn: - try: self._progress_finish_fn(automation_id, result) - except Exception: pass + try: + self._progress_finish_fn(automation_id, result) + except Exception as e: + logger.debug("scheduled progress finish: %s", e) # Execute then-actions (notifications + fire_signal) try: @@ -673,8 +689,8 @@ class AutomationEngine: delay = self._calc_delay_seconds(trigger_config) if delay: next_run_str = _utc_after(delay) - except Exception: - pass + except Exception as e: + logger.debug("next run calc failed: %s", e) last_result = json.dumps(result) if result else None self.db.update_automation_run(automation_id, next_run=next_run_str, error=error, last_result=last_result) @@ -682,8 +698,8 @@ class AutomationEngine: if self._history_record_fn: try: self._history_record_fn(automation_id, result) - except Exception: - pass + except Exception as e: + logger.debug("history record failed: %s", e) if self._running: self.schedule_automation(automation_id) diff --git a/core/connection_detect.py b/core/connection_detect.py index fd23361f..ab19e3a9 100644 --- a/core/connection_detect.py +++ b/core/connection_detect.py @@ -79,9 +79,9 @@ def run_detection(server_type): api_response = requests.get(api_url, timeout=1) if api_response.status_code == 200 and 'MediaContainer' in api_response.text: return f"http://{ip}:{port}" - - except: - pass + + except Exception as e: + logger.debug("plex probe %s: %s", ip, e) return None def test_jellyfin_server(ip, port=8096): @@ -101,9 +101,9 @@ def run_detection(server_type): web_response = requests.get(web_url, timeout=1) if web_response.status_code == 200 and 'jellyfin' in web_response.text.lower(): return f"http://{ip}:{port}" - - except: - pass + + except Exception as e: + logger.debug("jellyfin probe %s: %s", ip, e) return None def test_slskd_server(ip, port=5030): @@ -117,8 +117,8 @@ def run_detection(server_type): if response.status_code in [200, 401]: return f"http://{ip}:{port}" - except: - pass + except Exception as e: + logger.debug("slskd probe %s: %s", ip, e) return None def test_navidrome_server(ip, port=4533): @@ -140,8 +140,8 @@ def run_detection(server_type): # Check for Subsonic/Navidrome API response structure if 'subsonic-response' in data: return f"http://{ip}:{port}" - except: - pass + except Exception as e: + logger.debug("navidrome json parse: %s", e) # Also try the web interface web_url = f"http://{ip}:{port}/" @@ -149,8 +149,8 @@ def run_detection(server_type): if web_response.status_code == 200 and 'navidrome' in web_response.text.lower(): return f"http://{ip}:{port}" - except: - pass + except Exception as e: + logger.debug("navidrome probe %s: %s", ip, e) return None try: diff --git a/core/connection_test.py b/core/connection_test.py index 9401d1aa..4064bab5 100644 --- a/core/connection_test.py +++ b/core/connection_test.py @@ -1,6 +1,6 @@ """Service connection test — lifted from web_server.py. -The function body is byte-identical to the original. soulseek_client, +The function body is byte-identical to the original. download_orchestrator, qobuz_enrichment_worker, hydrabase_client, docker_resolve_url, and docker_resolve_path are injected at runtime because they live in web_server.py and are constructed there. @@ -27,7 +27,7 @@ def _get_metadata_fallback_source(): # Injected at runtime via init(). -soulseek_client = None +download_orchestrator = None qobuz_enrichment_worker = None hydrabase_client = None docker_resolve_url = None @@ -35,16 +35,16 @@ docker_resolve_path = None def init( - soulseek_client_obj, + download_orchestrator_obj, qobuz_worker, hydrabase_client_obj, docker_resolve_url_fn, docker_resolve_path_fn, ): """Bind web_server-side helpers/globals so the lifted body can resolve them.""" - global soulseek_client, qobuz_enrichment_worker, hydrabase_client + global download_orchestrator, qobuz_enrichment_worker, hydrabase_client global docker_resolve_url, docker_resolve_path - soulseek_client = soulseek_client_obj + download_orchestrator = download_orchestrator_obj qobuz_enrichment_worker = qobuz_worker hydrabase_client = hydrabase_client_obj docker_resolve_url = docker_resolve_url_fn @@ -177,13 +177,13 @@ def run_service_test(service, test_config): else: return False, f"Output folder not found: {transfer_path}" elif service == "soulseek": - if soulseek_client is None: + if download_orchestrator is None: return False, "Download orchestrator failed to initialize. Check server logs for startup errors." # Test the orchestrator's configured download source (not just Soulseek) download_mode = config_manager.get('download_source.mode', 'hybrid') - if run_async(soulseek_client.check_connection()): + if run_async(download_orchestrator.check_connection()): # Success message based on active mode mode_messages = { 'soulseek': "Successfully connected to Soulseek network via slskd.", @@ -379,6 +379,24 @@ def run_service_test(service, test_config): return False, "Hydrabase not connected. Configure URL + API key and click Connect." except Exception as e: return False, f"Hydrabase connection error: {str(e)}" + elif service == "soundcloud": + # Anonymous SoundCloud has no auth, so "test" really means + # "is yt-dlp installed and can it reach SoundCloud right now." + # This mirrors the /api/soundcloud/status check. + try: + from core.soundcloud_client import SoundcloudClient + sc = SoundcloudClient() + if not sc.is_available(): + return False, "SoundCloud unavailable — yt-dlp not installed." + # Run a tiny live probe via asyncio so the dashboard test + # gives a meaningful pass/fail. + import asyncio + reachable = asyncio.new_event_loop().run_until_complete(sc.check_connection()) + if reachable: + return True, "SoundCloud reachable (anonymous)" + return False, "SoundCloud unreachable — search probe failed. Try again." + except Exception as e: + return False, f"SoundCloud connection error: {str(e)}" return False, "Unknown service." except AttributeError as e: # This specifically catches the error you reported for Jellyfin diff --git a/core/database_update_worker.py b/core/database_update_worker.py index 3037fa79..3262ee30 100644 --- a/core/database_update_worker.py +++ b/core/database_update_worker.py @@ -407,9 +407,14 @@ class DatabaseUpdateWorker: logger.error(f"Could not connect to {self.server_type} server — check URL, credentials, and network (Docker users: use container name or host.docker.internal instead of host IP)") return [] - # Check for music library (Plex-specific check) - if self.server_type == "plex" and not self.media_client.music_library: - logger.error("No music library found in Plex") + # Check for music library (Plex-specific check). Routes + # through ``is_fully_configured`` so all-libraries mode (in + # which ``music_library`` is None but ``_all_libraries_mode`` + # is True) counts as configured. Pre-fix this bailed out on + # the bare music_library None check, silently aborting the + # deep scan for any all-libraries-mode user. + if self.server_type == "plex" and not self.media_client.is_fully_configured(): + logger.error("No music library configured in Plex") return [] # Check if database has enough content for incremental updates (server-specific) @@ -1046,8 +1051,8 @@ class DatabaseUpdateWorker: batch + [self.server_type]) cascade_album_ids.update(row[0] for row in cursor.fetchall()) removed_album_ids -= cascade_album_ids - except Exception: - pass # If this optimization fails, double-delete is harmless + except Exception as e: + logger.debug("cascade album cleanup optimization: %s", e) if not removed_artist_ids and not removed_album_ids: logger.info("Removal detection: no stale content found") @@ -1084,24 +1089,31 @@ class DatabaseUpdateWorker: return [] def _get_recent_albums_plex(self) -> List: - """Get recently added and updated albums from Plex""" + """Get recently added and updated albums from Plex. + + Routes through ``PlexClient.get_recently_added_albums`` and + ``get_recently_updated_albums`` so the all-libraries mode union + works (pre-fix this reached ``self.media_client.music_library.X`` + directly which crashed when music_library is None in all- + libraries mode). + """ all_recent_content = [] - + try: - # Get recently added albums (up to 400 to catch more recent content) - try: - recently_added = self.media_client.music_library.recentlyAdded(libtype='album', maxresults=400) + # Get recently added albums (up to 400 to catch more recent content) + recently_added = self.media_client.get_recently_added_albums(maxresults=400, libtype='album') + if recently_added: all_recent_content.extend(recently_added) logger.info(f"Found {len(recently_added)} recently added albums") - except: - # Fallback to general recently added - recently_added = self.media_client.music_library.recentlyAdded(maxresults=400) - all_recent_content.extend(recently_added) - logger.info(f"Found {len(recently_added)} recently added items (mixed types)") - + else: + # Fallback to mixed-type recents. + recently_added = self.media_client.get_recently_added_albums(maxresults=400, libtype=None) + all_recent_content.extend(recently_added or []) + logger.info(f"Found {len(recently_added or [])} recently added items (mixed types)") + # Get recently updated albums (catches metadata corrections) try: - recently_updated = self.media_client.music_library.search(sort='updatedAt:desc', libtype='album', limit=400) + recently_updated = self.media_client.get_recently_updated_albums(limit=400) # Remove duplicates (items that are both recently added and updated) added_keys = {getattr(item, 'ratingKey', None) for item in all_recent_content} unique_updated = [item for item in recently_updated if getattr(item, 'ratingKey', None) not in added_keys] diff --git a/core/debug_info.py b/core/debug_info.py index 4ad83f1d..88a5ece9 100644 --- a/core/debug_info.py +++ b/core/debug_info.py @@ -64,7 +64,7 @@ download_batches = None sync_states = None youtube_playlist_states = None tidal_discovery_states = None -soulseek_client = None +download_orchestrator = None _log_path = None _log_dir = None app = None @@ -80,7 +80,7 @@ def init( sync_states_dict, youtube_playlist_states_dict, tidal_discovery_states_dict, - soulseek_client_obj, + download_orchestrator_obj, log_path, log_dir, flask_app, @@ -90,7 +90,7 @@ def init( """Bind shared state/helpers from web_server.""" global SOULSYNC_VERSION, _DIRECT_RUN, _status_cache, qobuz_enrichment_worker global download_batches, sync_states, youtube_playlist_states - global tidal_discovery_states, soulseek_client, _log_path, _log_dir + global tidal_discovery_states, download_orchestrator, _log_path, _log_dir global app, get_database, _get_tidal_client SOULSYNC_VERSION = soulsync_version _DIRECT_RUN = direct_run @@ -100,7 +100,7 @@ def init( sync_states = sync_states_dict youtube_playlist_states = youtube_playlist_states_dict tidal_discovery_states = tidal_discovery_states_dict - soulseek_client = soulseek_client_obj + download_orchestrator = download_orchestrator_obj _log_path = log_path _log_dir = log_dir app = flask_app @@ -260,8 +260,8 @@ def get_debug_info(): for _pid, st in list(tidal_discovery_states.items()): if st.get('phase') == 'syncing': active_syncs += 1 - except Exception: - pass + except Exception as e: + logger.debug("count active syncs failed: %s", e) info['active_downloads'] = active_downloads info['active_syncs'] = active_syncs @@ -292,31 +292,32 @@ def get_debug_info(): # Download client init failures info['download_client_failures'] = [] - if soulseek_client and hasattr(soulseek_client, '_init_failures'): - info['download_client_failures'] = soulseek_client._init_failures - elif not soulseek_client: + if download_orchestrator and hasattr(download_orchestrator, '_init_failures'): + info['download_client_failures'] = download_orchestrator._init_failures + elif not download_orchestrator: info['download_client_failures'] = ['ALL (orchestrator failed to initialize)'] # API rate monitor — current calls/min, 24h totals, peaks, rate limit events try: from core.api_call_tracker import api_call_tracker + from core.metadata.status import get_spotify_status rates = api_call_tracker.get_all_rates() info['api_rates'] = rates # Rich 24h debug summary with peaks, totals, per-endpoint breakdown, events info['api_debug_summary'] = api_call_tracker.get_debug_summary() # Spotify rate limit details - if spotify_client: - rl_info = spotify_client.get_rate_limit_info() - if rl_info: - info['spotify_rate_limit'] = { - 'active': True, - 'remaining_seconds': rl_info.get('remaining_seconds', 0), - 'retry_after': rl_info.get('retry_after', 0), - 'endpoint': rl_info.get('endpoint', ''), - 'expires_at': rl_info.get('expires_at', ''), - } - else: - info['spotify_rate_limit'] = {'active': False} + spotify_status = get_spotify_status(spotify_client=spotify_client) + rl_info = spotify_status.get('rate_limit') + if spotify_status.get('rate_limited') and rl_info: + info['spotify_rate_limit'] = { + 'active': True, + 'remaining_seconds': rl_info.get('remaining_seconds', 0), + 'retry_after': rl_info.get('retry_after', 0), + 'endpoint': rl_info.get('endpoint', ''), + 'expires_at': rl_info.get('expires_at', ''), + } + else: + info['spotify_rate_limit'] = {'active': False} except Exception: info['api_rates'] = {} info['api_debug_summary'] = {} diff --git a/core/deezer_client.py b/core/deezer_client.py index 6b2a26c1..59330239 100644 --- a/core/deezer_client.py +++ b/core/deezer_client.py @@ -307,8 +307,8 @@ class DeezerClient: for raw in cached_results: try: tracks.append(Track.from_deezer_track(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Track.from_deezer_track cache parse: %s", e) if tracks: return tracks @@ -341,8 +341,8 @@ class DeezerClient: for raw in cached_results: try: artists.append(Artist.from_deezer_artist(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Artist.from_deezer_artist cache parse: %s", e) if artists: return artists @@ -375,8 +375,8 @@ class DeezerClient: for raw in cached_results: try: albums.append(Album.from_deezer_album(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Album.from_deezer_album cache parse: %s", e) if albums: return albums @@ -599,6 +599,74 @@ class DeezerClient: return result + def get_artist_top_tracks(self, artist_id: str, limit: int = 10) -> List[Dict[str, Any]]: + """Return the artist's top tracks in Spotify-compatible dict format. + + Wraps Deezer's `/artist/{id}/top?limit=N`. Returns dicts with the same + shape Spotify's `artist_top_tracks` produces — id, name, artists, album + (with album_type / total_tracks / release_date / images), duration_ms, + track_number, disc_number — so callers don't need to branch on source. + """ + if not artist_id: + return [] + try: + limit = max(1, min(int(limit or 10), 100)) + except (TypeError, ValueError): + limit = 10 + + data = self._api_get(f'artist/{artist_id}/top', {'limit': limit}) + if not data or 'data' not in data: + return [] + + tracks = [] + for track_data in data['data']: + if not isinstance(track_data, dict): + continue + artist_data = track_data.get('artist') or {} + album_data = track_data.get('album') or {} + + # Build images list from any cover sizes Deezer returned for the album + images = [] + if isinstance(album_data, dict): + for size_key, dim in [('cover_xl', 1000), ('cover_big', 500), + ('cover_medium', 250), ('cover_small', 56)]: + if album_data.get(size_key): + images.append({'url': album_data[size_key], 'height': dim, 'width': dim}) + + # Deezer `/artist/{id}/top` results don't include record_type on the + # nested album object; we don't have a track-count to infer from + # either. Default 'album' so the path-builder template variable + # always has something to substitute (existing behavior elsewhere). + album_payload = { + 'id': str(album_data.get('id', '')) if isinstance(album_data, dict) else '', + 'name': album_data.get('title', '') if isinstance(album_data, dict) else '', + 'album_type': 'album', + 'images': images, + 'release_date': '', + 'total_tracks': 0, + 'artists': [{'name': artist_data.get('name', '')}] if isinstance(artist_data, dict) else [], + } + + tracks.append({ + 'id': str(track_data.get('id', '')), + 'name': track_data.get('title', ''), + 'artists': [{ + 'id': str(artist_data.get('id', '')) if isinstance(artist_data, dict) else '', + 'name': artist_data.get('name', '') if isinstance(artist_data, dict) else '', + }], + 'album': album_payload, + 'duration_ms': (track_data.get('duration') or 0) * 1000, # Deezer is seconds + 'popularity': track_data.get('rank', 0), + 'preview_url': track_data.get('preview'), + 'external_urls': {'deezer': track_data['link']} if track_data.get('link') else {}, + 'track_number': track_data.get('track_position'), + 'disc_number': track_data.get('disk_number', 1), + 'explicit': bool(track_data.get('explicit_lyrics', False)), + '_source': 'deezer', + }) + + return tracks + def get_artist_info(self, artist_id: str) -> Optional[Dict[str, Any]]: """Get full artist details — returns Spotify-compatible dict (metadata source interface). @@ -842,8 +910,8 @@ class DeezerClient: try: cache = get_metadata_cache() cache.store_entity('deezer', 'artist', str(result.get('id', '')), result) - except Exception: - pass + except Exception as e: + logger.debug("cache store_entity artist search: %s", e) logger.debug(f"Found artist for query: {artist_name}") return result @@ -887,8 +955,8 @@ class DeezerClient: try: cache = get_metadata_cache() cache.store_entity('deezer', 'album', str(result.get('id', '')), result) - except Exception: - pass + except Exception as e: + logger.debug("cache store_entity album search: %s", e) logger.debug(f"Found album for query: {artist_name} - {album_title}") return result @@ -932,8 +1000,8 @@ class DeezerClient: try: cache = get_metadata_cache() cache.store_entity('deezer', 'track', str(result.get('id', '')), result) - except Exception: - pass + except Exception as e: + logger.debug("cache store_entity track search: %s", e) logger.debug(f"Found track for query: {artist_name} - {track_title}") return result @@ -965,8 +1033,8 @@ class DeezerClient: # Cache hit with full details (has label = was a get_album response, not just search) logger.debug(f"Cache hit for album {album_id}") return cached - except Exception: - pass + except Exception as e: + logger.debug("cache get_entity album: %s", e) try: response = self.session.get( @@ -984,8 +1052,8 @@ class DeezerClient: try: cache = get_metadata_cache() cache.store_entity('deezer', 'album', str(album_id), data) - except Exception: - pass + except Exception as e: + logger.debug("cache store_entity album full: %s", e) logger.debug(f"Got full album details for ID: {album_id}") return data @@ -1013,8 +1081,8 @@ class DeezerClient: if cached and cached.get('bpm'): logger.debug(f"Cache hit for track {track_id}") return cached - except Exception: - pass + except Exception as e: + logger.debug("cache get_entity track: %s", e) try: response = self.session.get( @@ -1032,8 +1100,8 @@ class DeezerClient: try: cache = get_metadata_cache() cache.store_entity('deezer', 'track', str(track_id), data) - except Exception: - pass + except Exception as e: + logger.debug("cache store_entity track full: %s", e) logger.debug(f"Got full track details for ID: {track_id}") return data diff --git a/core/deezer_download_client.py b/core/deezer_download_client.py index beaf9c01..727c7006 100644 --- a/core/deezer_download_client.py +++ b/core/deezer_download_client.py @@ -20,7 +20,7 @@ from typing import Any, Dict, List, Optional, Tuple import requests -from core.soulseek_client import AlbumResult, DownloadStatus, TrackResult +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult from utils.logging_config import get_logger logger = get_logger("deezer_download") @@ -79,7 +79,10 @@ def _decrypt_chunk(chunk: bytes, key: bytes) -> bytes: ) from exc -class DeezerDownloadClient: +from core.download_plugins.base import DownloadSourcePlugin + + +class DeezerDownloadClient(DownloadSourcePlugin): """Deezer download client using ARL token authentication.""" def __init__(self, download_path: str = None): @@ -91,9 +94,9 @@ class DeezerDownloadClient: self.download_path = Path(download_path) self.download_path.mkdir(parents=True, exist_ok=True) - # Download tracking (same pattern as Tidal/Qobuz/HiFi) - self.active_downloads: Dict[str, Dict[str, Any]] = {} - self._download_lock = threading.Lock() + # Engine reference is populated by set_engine() at registration + # time. None until orchestrator wires the registry. + self._engine = None # Shutdown check callback (set by web_server) self.shutdown_check = None @@ -125,6 +128,10 @@ class DeezerDownloadClient: logger.info(f"Deezer download client initialized (download path: {self.download_path})") + def set_engine(self, engine): + """Engine callback — wires the central thread worker + state store.""" + self._engine = engine + # ─── Authentication ────────────────────────────────────────── def _authenticate(self, arl: str) -> bool: @@ -423,8 +430,8 @@ class DeezerDownloadClient: if cached and cached.get('release_date'): album_release_dates[aid] = cached['release_date'] continue - except Exception: - pass + except Exception as e: + logger.debug("cache get_entity album release_date: %s", e) # Cache miss — fetch from API try: time.sleep(0.3) # Respect rate limits @@ -436,10 +443,10 @@ class DeezerDownloadClient: if cache: try: cache.store_entity('deezer', 'album', aid, a_data) - except Exception: - pass - except Exception: - pass + except Exception as e: + logger.debug("cache store_entity album release_date: %s", e) + except Exception as e: + logger.debug("fetch deezer album release_date %s: %s", aid, e) tracks = [] for i, t in enumerate(raw_tracks, start=1): @@ -605,87 +612,67 @@ class DeezerDownloadClient: if not self._authenticated: logger.error("Deezer not authenticated — cannot download") return None + if self._engine is None: + # Raise rather than return None so the orchestrator's + # download_with_fallback surfaces a real warning + tries + # the next source. Returning None silently dropped the + # download with no user feedback (per JohnBaumb). + raise RuntimeError("Deezer client has no engine reference — cannot dispatch download") # Parse filename: "track_id||display_name" parts = filename.split('||', 1) track_id = parts[0] display_name = parts[1] if len(parts) > 1 else f"Track {track_id}" - download_id = str(uuid.uuid4()) - - with self._download_lock: - self.active_downloads[download_id] = { - 'id': download_id, + return self._engine.worker.dispatch( + source_name='deezer', + target_id=track_id, + display_name=display_name, + original_filename=filename, + impl_callable=self._download_sync, + extra_record_fields={ 'track_id': track_id, 'display_name': display_name, - 'filename': filename, - 'username': 'deezer_dl', - 'state': 'Initializing', - 'progress': 0.0, 'size': file_size, - 'transferred': 0, - 'speed': 0, - 'file_path': None, 'error': None, - } - - thread = threading.Thread( - target=self._download_thread_worker, - args=(download_id, track_id, display_name), - daemon=True, - name=f'deezer-dl-{track_id}' + }, + # Legacy username slot — frontend status indicators key off + # ``deezer_dl``, not the canonical ``deezer``. + username_override='deezer_dl', + # Diagnostic thread name for multi-thread debugging. + thread_name=f'deezer-dl-{track_id}', ) - thread.start() - logger.info(f"Started Deezer download {download_id}: {display_name}") - return download_id + def _set_error(self, download_id: str, message: str) -> None: + """Helper: set the engine record's `error` slot. No-op if + engine isn't wired or record was already removed.""" + if self._engine is None: + return + self._engine.update_record('deezer', download_id, {'error': message}) - def _download_thread_worker(self, download_id: str, track_id: str, display_name: str): - """Background worker for a single download.""" - try: - result_path = self._download_sync(download_id, track_id, display_name) - with self._download_lock: - if download_id in self.active_downloads: - dl = self.active_downloads[download_id] - if dl['state'] == 'Cancelled': - return - if result_path: - dl['state'] = 'Completed, Succeeded' - dl['progress'] = 100.0 - dl['file_path'] = result_path - logger.info(f"Deezer download {download_id} completed: {result_path}") - else: - dl['state'] = 'Errored' - logger.error(f"Deezer download {download_id} failed: {dl.get('error', 'unknown')}") - except Exception as e: - logger.error(f"Deezer download thread error: {e}") - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' - self.active_downloads[download_id]['error'] = str(e) + def _is_cancelled(self, download_id: str) -> bool: + if self._engine is None: + return False + record = self._engine.get_record('deezer', download_id) + return record is not None and record.get('state') == 'Cancelled' def _download_sync(self, download_id: str, track_id: str, display_name: str) -> Optional[str]: """Synchronous download: get URL, download, decrypt, save.""" # Check for shutdown if self.shutdown_check and self.shutdown_check(): - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Aborted' + if self._engine is not None: + self._engine.update_record('deezer', download_id, {'state': 'Aborted'}) return None # Get track data from private API track_data = self._get_track_data(track_id) if not track_data: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['error'] = 'Failed to get track data' + self._set_error(download_id, 'Failed to get track data') return None track_token = track_data.get('TRACK_TOKEN', '') if not track_token: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['error'] = 'No track token available' + self._set_error(download_id, 'No track token available') return None # Determine quality and get media URL with fallback @@ -695,7 +682,6 @@ class DeezerDownloadClient: if allow_fallback: quality_order = _QUALITY_ORDER.copy() - # Start from user's preferred quality try: pref_idx = quality_order.index(self._quality) quality_order = quality_order[pref_idx:] + quality_order[:pref_idx] @@ -712,27 +698,16 @@ class DeezerDownloadClient: break if not media_url: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['error'] = 'No media URL available (may require higher subscription tier)' + self._set_error(download_id, 'No media URL available (may require higher subscription tier)') return None if actual_quality != self._quality: logger.info(f"Quality fallback: {self._quality} → {actual_quality} for {display_name}") - # Determine file extension ext = '.flac' if actual_quality == 'flac' else '.mp3' - - # Sanitize filename safe_name = self._sanitize_filename(display_name) out_path = str(self.download_path / f"{safe_name}{ext}") - # Update state - with self._download_lock: - if download_id in self.active_downloads: - dl = self.active_downloads[download_id] - dl['state'] = 'InProgress, Downloading' - # Download and decrypt try: bf_key = _get_blowfish_key(track_id) @@ -740,9 +715,8 @@ class DeezerDownloadClient: resp.raise_for_status() total_size = int(resp.headers.get('content-length', 0)) - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['size'] = total_size + if self._engine is not None: + self._engine.update_record('deezer', download_id, {'size': total_size}) downloaded = 0 chunk_index = 0 @@ -755,23 +729,20 @@ class DeezerDownloadClient: # Check for cancellation/shutdown if self.shutdown_check and self.shutdown_check(): - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Aborted' + if self._engine is not None: + self._engine.update_record('deezer', download_id, {'state': 'Aborted'}) try: os.remove(out_path) except OSError: pass return None - with self._download_lock: - if download_id in self.active_downloads: - if self.active_downloads[download_id]['state'] == 'Cancelled': - try: - os.remove(out_path) - except OSError: - pass - return None + if self._is_cancelled(download_id): + try: + os.remove(out_path) + except OSError: + pass + return None # Decrypt every 3rd chunk (Deezer's encryption pattern) if chunk_index % 3 == 0 and len(raw_chunk) == _CHUNK_SIZE: @@ -788,12 +759,12 @@ class DeezerDownloadClient: speed = int(downloaded / elapsed) if elapsed > 0 else 0 progress = (downloaded / total_size * 100) if total_size > 0 else 0 - with self._download_lock: - if download_id in self.active_downloads: - dl = self.active_downloads[download_id] - dl['transferred'] = downloaded - dl['progress'] = min(progress, 99.9) - dl['speed'] = speed + if self._engine is not None: + self._engine.update_record('deezer', download_id, { + 'transferred': downloaded, + 'progress': min(progress, 99.9), + 'speed': speed, + }) # Validate file size file_size = os.path.getsize(out_path) @@ -803,9 +774,7 @@ class DeezerDownloadClient: os.remove(out_path) except OSError: pass - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['error'] = f'File too small ({file_size} bytes)' + self._set_error(download_id, f'File too small ({file_size} bytes)') return None logger.info(f"Deezer download complete: {out_path} ({file_size / 1048576:.1f} MB, {actual_quality})") @@ -817,59 +786,58 @@ class DeezerDownloadClient: os.remove(out_path) except OSError: pass - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['error'] = str(e) + self._set_error(download_id, str(e)) return None # ─── Download Status ───────────────────────────────────────── + def _record_to_status(self, record: dict) -> DownloadStatus: + return DownloadStatus( + id=record['id'], + filename=record['filename'], + username=record['username'], + state=record['state'], + progress=record['progress'], + size=record.get('size', 0), + transferred=record.get('transferred', 0), + speed=record.get('speed', 0), + file_path=record.get('file_path'), + ) + async def get_all_downloads(self) -> List[DownloadStatus]: - """Return all active downloads.""" - with self._download_lock: - return [self._to_status(dl) for dl in self.active_downloads.values()] + if self._engine is None: + return [] + return [ + self._record_to_status(record) + for record in self._engine.iter_records_for_source('deezer') + ] async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: - """Get status of a specific download.""" - with self._download_lock: - dl = self.active_downloads.get(download_id) - return self._to_status(dl) if dl else None + if self._engine is None: + return None + record = self._engine.get_record('deezer', download_id) + return self._record_to_status(record) if record is not None else None async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool: - """Cancel a download.""" - with self._download_lock: - dl = self.active_downloads.get(download_id) - if not dl: - return False - dl['state'] = 'Cancelled' - if remove: - del self.active_downloads[download_id] + if self._engine is None: + return False + if self._engine.get_record('deezer', download_id) is None: + return False + self._engine.update_record('deezer', download_id, {'state': 'Cancelled'}) + if remove: + self._engine.remove_record('deezer', download_id) return True async def clear_all_completed_downloads(self) -> bool: - """Remove all terminal downloads.""" - terminal_states = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} - with self._download_lock: - to_remove = [k for k, v in self.active_downloads.items() if v['state'] in terminal_states] - for k in to_remove: - del self.active_downloads[k] + if self._engine is None: + return True + terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} + for record in list(self._engine.iter_records_for_source('deezer')): + if record.get('state') in terminal: + self._engine.remove_record('deezer', record['id']) return True - def _to_status(self, dl: dict) -> DownloadStatus: - """Convert internal dict to DownloadStatus.""" - return DownloadStatus( - id=dl['id'], - filename=dl['filename'], - username=dl['username'], - state=dl['state'], - progress=dl['progress'], - size=dl['size'], - transferred=dl['transferred'], - speed=dl['speed'], - file_path=dl.get('file_path'), - ) - # ─── Utilities ─────────────────────────────────────────────── @staticmethod diff --git a/core/deezer_worker.py b/core/deezer_worker.py index 4940d16d..aba5c6fb 100644 --- a/core/deezer_worker.py +++ b/core/deezer_worker.py @@ -9,6 +9,7 @@ from utils.logging_config import get_logger from database.music_database import MusicDatabase from core.deezer_client import DeezerClient from core.worker_utils import interruptible_sleep, set_album_api_track_count +from core.enrichment.manual_match_honoring import honor_stored_match logger = get_logger("deezer_worker") @@ -140,8 +141,8 @@ class DeezerWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null id table resolve failed: %s", e) continue @@ -383,11 +384,33 @@ class DeezerWorker: self.stats['not_found'] += 1 logger.debug(f"No match for artist '{artist_name}'") + def _refresh_album_via_stored_id(self, album_id, stored_id, full_album_dict): + """Issue #501 callback. Stored ID exists → fetched full Deezer + album payload. Use it as both args to ``_update_album`` (search- + result and full-data shapes overlap on the fields we need — + artist verification skipped since manual match presumably + already vetted).""" + self._update_album(album_id, full_album_dict, full_album_dict) + + def _refresh_track_via_stored_id(self, track_id, stored_id, full_track_dict): + """Issue #501 callback for tracks — same pattern as albums.""" + self._update_track(track_id, full_track_dict, full_track_dict) + def _process_album(self, album_id: int, album_name: str, artist_name: str, item: Dict[str, Any]): """Process an album: search Deezer, verify, fetch full details, store metadata""" - existing_id = self._get_existing_id('album', album_id) - if existing_id: - logger.debug(f"Preserving existing Deezer ID for album '{album_name}': {existing_id}") + # Issue #501: honor manual matches. Pre-fix this method just + # SKIPPED when a stored ID was present (preserved the ID but + # never refreshed metadata). Now it goes through the full + # refresh path via the stored ID, picking up label / genres / + # explicit updates without ever overwriting the manual match. + if honor_stored_match( + db=self.db, entity_table='albums', entity_id=album_id, + id_column='deezer_id', + client_fetch_fn=self.client.get_album_raw, + on_match_fn=self._refresh_album_via_stored_id, + log_prefix='Deezer', + ): + self.stats['matched'] += 1 return result = self.client.search_album(artist_name, album_name) @@ -430,9 +453,15 @@ class DeezerWorker: def _process_track(self, track_id: int, track_name: str, artist_name: str, item: Dict[str, Any]): """Process a track: search Deezer, verify, fetch full details for BPM, store metadata""" - existing_id = self._get_existing_id('track', track_id) - if existing_id: - logger.debug(f"Preserving existing Deezer ID for track '{track_name}': {existing_id}") + # Issue #501: honor manual matches (see _process_album). + if honor_stored_match( + db=self.db, entity_table='tracks', entity_id=track_id, + id_column='deezer_id', + client_fetch_fn=self.client.get_track_raw, + on_match_fn=self._refresh_track_via_stored_id, + log_prefix='Deezer', + ): + self.stats['matched'] += 1 return result = self.client.search_track(artist_name, track_name) diff --git a/core/discogs_client.py b/core/discogs_client.py index 351fe7bc..4147277d 100644 --- a/core/discogs_client.py +++ b/core/discogs_client.py @@ -320,8 +320,8 @@ class DiscogsClient: try: from config.settings import config_manager self.token = config_manager.get('discogs.token', '') - except Exception: - pass + except Exception as e: + logger.debug("load discogs.token from config: %s", e) if self.token: self.session.headers['Authorization'] = f'Discogs token={self.token}' @@ -369,6 +369,125 @@ class DiscogsClient: logger.error(f"Discogs API error ({endpoint}): {e}") return None + # --- User Collection (powers Your Albums Discogs source) --- + + def get_authenticated_username(self) -> Optional[str]: + """Resolve the username for the configured personal token. + + Discogs's `/oauth/identity` endpoint returns the user's + username when called with a valid token. Cached on the + instance so subsequent calls don't re-hit the API. + """ + if hasattr(self, '_cached_username'): + return self._cached_username + if not self.is_authenticated(): + self._cached_username = None + return None + data = self._api_get('/oauth/identity') + username = data.get('username') if data else None + self._cached_username = username + return username + + def get_user_collection(self, username: Optional[str] = None, + folder_id: int = 0, + per_page: int = 100, + max_pages: int = 50) -> List[Dict[str, Any]]: + """Fetch a Discogs user's collection (folder 0 = "All"). + + Returns a list of normalized release dicts ready for + ``database.upsert_liked_album``: + { + 'album_name': str, + 'artist_name': str, + 'release_id': int, # Discogs release id + 'image_url': str | None, + 'release_date': str, # 'YYYY' (Discogs only stores year) + 'total_tracks': int, + } + + Pagination caps at ``max_pages`` to bound runtime — at 100/page + that's 5000 releases, more than enough for typical collections. + Authenticated calls only (Discogs collection is private). + """ + if not self.is_authenticated(): + logger.warning("Discogs collection fetch attempted without token") + return [] + + if not username: + username = self.get_authenticated_username() + if not username: + logger.warning("Could not resolve Discogs username for token") + return [] + + results: List[Dict[str, Any]] = [] + page = 1 + while page <= max_pages: + data = self._api_get( + f'/users/{username}/collection/folders/{folder_id}/releases', + {'page': page, 'per_page': per_page, 'sort': 'added', 'sort_order': 'desc'}, + ) + if not data: + break + + releases = data.get('releases', []) or [] + if not releases: + break + + for entry in releases: + info = entry.get('basic_information') or {} + release_id = entry.get('id') or info.get('id') + if not release_id: + continue + title = info.get('title') or '' + # Discogs `artists` is a list of {name, id, ...}; first is primary. + artists = info.get('artists') or [] + artist_name = '' + if artists and isinstance(artists[0], dict): + artist_name = (artists[0].get('name') or '').strip() + # Strip trailing "(N)" disambiguation suffix Discogs adds. + artist_name = re.sub(r'\s*\(\d+\)$', '', artist_name) + if not title or not artist_name: + continue + + # Image URLs: cover_image is the primary, also has thumb. + image_url = (info.get('cover_image') + or info.get('thumb') + or '') + + year = info.get('year') + release_date = str(year) if year and year > 0 else '' + + results.append({ + 'album_name': title.strip(), + 'artist_name': artist_name, + 'release_id': int(release_id), + 'image_url': image_url or None, + 'release_date': release_date, + 'total_tracks': 0, # Not in basic_information; populated via get_release if needed + }) + + pagination = data.get('pagination') or {} + if page >= int(pagination.get('pages') or 1): + break + page += 1 + + logger.info(f"Discogs collection: fetched {len(results)} releases for {username}") + return results + + def get_release(self, release_id: int) -> Optional[Dict[str, Any]]: + """Fetch full Discogs release detail including tracklist. + + Returns the raw API response so callers can render rich + Discogs context (year, format, label, country, tracklist). + """ + if not release_id: + return None + try: + release_id = int(release_id) + except (TypeError, ValueError): + return None + return self._api_get(f'/releases/{release_id}') + # --- Search Methods (same signatures as iTunes/Deezer) --- def search_artists(self, query: str, limit: int = 10) -> List[Artist]: @@ -380,8 +499,8 @@ class DiscogsClient: for raw in cached_results: try: artists.append(Artist.from_discogs_artist(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Artist.from_discogs_artist cache parse: %s", e) if artists: return artists @@ -417,8 +536,8 @@ class DiscogsClient: for raw in cached_results: try: albums.append(Album.from_discogs_release(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Album.from_discogs_release cache parse: %s", e) if albums: return albums diff --git a/core/discogs_worker.py b/core/discogs_worker.py index 84270de9..8663934f 100644 --- a/core/discogs_worker.py +++ b/core/discogs_worker.py @@ -298,8 +298,8 @@ class DiscogsWorker: self.stats['errors'] += 1 try: self._mark_status(item['type'], item['id'], 'error') - except Exception: - pass + except Exception as e: + logger.debug("mark item status error failed: %s", e) def _get_existing_id(self, entity_type: str, entity_id) -> Optional[str]: """Check if entity already has a discogs_id.""" diff --git a/core/discovery/playlist.py b/core/discovery/playlist.py index 5386dd49..4bbbf1c7 100644 --- a/core/discovery/playlist.py +++ b/core/discovery/playlist.py @@ -196,8 +196,8 @@ def run_playlist_discovery_worker(playlists, automation_id=None, deps: PlaylistD current_item=track_name, log_line=f'{track_name} → {cached_match.get("name", "?")} (cache)', log_type='success') continue - except Exception: - pass + except Exception as e: + logger.debug("discovery cache lookup failed: %s", e) # Step 2: Generate search queries try: @@ -252,8 +252,8 @@ def run_playlist_discovery_worker(playlists, automation_id=None, deps: PlaylistD if match and confidence > best_confidence: best_confidence = confidence best_match = match - except Exception: - pass + except Exception as e: + logger.debug("extended discovery search failed: %s", e) # Step 4: Store results if best_match and best_confidence >= min_confidence: @@ -290,8 +290,8 @@ def run_playlist_discovery_worker(playlists, automation_id=None, deps: PlaylistD if _raw: track_number = _raw.get('track_number') disc_number = _raw.get('disc_number') - except Exception: - pass + except Exception as e: + logger.debug("metadata cache lookup for album enrichment failed: %s", e) matched_data = { 'id': best_match.id if hasattr(best_match, 'id') else '', @@ -323,8 +323,8 @@ def run_playlist_discovery_worker(playlists, automation_id=None, deps: PlaylistD best_confidence, matched_data, track_name, artist_name ) - except Exception: - pass + except Exception as e: + logger.debug("save discovery cache match failed: %s", e) logger.info(f"[{i+1}/{len(undiscovered_tracks)}] {track_name} → {matched_data['name']} ({best_confidence:.2f})") deps.update_automation_progress(automation_id, @@ -366,8 +366,8 @@ def run_playlist_discovery_worker(playlists, automation_id=None, deps: PlaylistD 'failed_count': str(total_failed), 'skipped_count': str(total_skipped), }) - except Exception: - pass + except Exception as e: + logger.debug("discovery_completed emit failed: %s", e) logger.error(f"Playlist discovery complete: {total_discovered} discovered, {total_failed} failed, {total_skipped} skipped") deps.update_automation_progress(automation_id, status='finished', progress=100, diff --git a/core/discovery/quality_scanner.py b/core/discovery/quality_scanner.py index 6c213c80..27457ca7 100644 --- a/core/discovery/quality_scanner.py +++ b/core/discovery/quality_scanner.py @@ -32,14 +32,30 @@ import logging import time from dataclasses import dataclass from datetime import datetime -from typing import Any, Callable +from typing import Any, Callable, Dict, Optional from core.metadata.registry import get_client_for_source, get_primary_source, get_source_priority +from core.metadata.types import Album from core.wishlist.payloads import ensure_wishlist_track_format logger = logging.getLogger(__name__) +# Per-source typed converter dispatch — same registry pattern as +# the metadata builders. Quality-scanner result normalization routes +# the embedded ``track.album`` blob through Album.from__dict() +# when provider is known. Falls back to legacy duck-typed extraction. +_TYPED_ALBUM_CONVERTERS: Dict[str, Callable[[Dict[str, Any]], Album]] = { + 'spotify': Album.from_spotify_dict, + 'itunes': Album.from_itunes_dict, + 'deezer': Album.from_deezer_dict, + 'discogs': Album.from_discogs_dict, + 'musicbrainz': Album.from_musicbrainz_dict, + 'hydrabase': Album.from_hydrabase_dict, + 'qobuz': Album.from_qobuz_dict, +} + + @dataclass class QualityScannerDeps: """Bundle of cross-cutting deps the quality scanner needs.""" @@ -140,7 +156,16 @@ def _normalize_image_entries(image_value: Any) -> list[dict]: return normalized -def _normalize_track_album(track_item: Any) -> dict: +def _normalize_track_album(track_item: Any, provider: Optional[str] = None) -> dict: + """Normalize a track's embedded album blob into a flat dict. + + When ``provider`` is provided AND maps to a registered typed Album + converter, routes through the typed path to seed canonical fields + on ``album_data`` before legacy fallback chains fill any gaps. + Falls back to legacy duck-typed extraction on unknown provider / + non-dict input / typed converter error — same pattern as the + metadata builders. + """ album = _extract_lookup_value(track_item, 'album', default={}) if isinstance(album, dict): album_data = dict(album) @@ -152,6 +177,27 @@ def _normalize_track_album(track_item: Any) -> dict: 'release_date': _extract_lookup_value(album, 'release_date', default='') or '', } + if provider and isinstance(album, dict): + converter = _TYPED_ALBUM_CONVERTERS.get(provider.strip().lower()) + if converter is not None: + try: + typed_album = converter(album) + if typed_album.name: + album_data.setdefault('name', typed_album.name) + if typed_album.album_type: + album_data.setdefault('album_type', typed_album.album_type) + if typed_album.total_tracks: + album_data.setdefault('total_tracks', typed_album.total_tracks) + if typed_album.release_date: + album_data.setdefault('release_date', typed_album.release_date) + if typed_album.id: + album_data.setdefault('id', typed_album.id) + except Exception as exc: + logger.debug( + "Typed album converter failed for provider %s in quality " + "scanner normalize, falling back to legacy: %s", provider, exc, + ) + album_data.setdefault('name', _extract_lookup_value(track_item, 'album_name', default='Unknown Album') or 'Unknown Album') album_data.setdefault('album_type', _extract_lookup_value(track_item, 'album_type', default='album') or 'album') album_data.setdefault('total_tracks', _extract_lookup_value(track_item, 'total_tracks', 'track_count', default=0) or 0) @@ -189,7 +235,7 @@ def _normalize_track_match(track_item: Any, provider: str) -> dict: 'id': _extract_lookup_value(track_item, 'id', 'track_id', default='') or '', 'name': _extract_lookup_value(track_item, 'name', 'title', default='Unknown Track') or 'Unknown Track', 'artists': _normalize_track_artists(track_item), - 'album': _normalize_track_album(track_item), + 'album': _normalize_track_album(track_item, provider=provider), 'image_url': _extract_lookup_value(track_item, 'image_url', 'album_cover_url', default=None), 'duration_ms': _extract_lookup_value(track_item, 'duration_ms', default=0) or 0, 'track_number': _extract_lookup_value(track_item, 'track_number', default=1) or 1, @@ -602,8 +648,8 @@ def run_quality_scanner(scope='watchlist', profile_id=1, deps: QualityScannerDep 'low_quality': str(deps.quality_scanner_state.get('low_quality', 0)), 'total_scanned': str(deps.quality_scanner_state.get('processed', 0)), }) - except Exception: - pass + except Exception as e: + logger.debug("emit quality_scan_completed failed: %s", e) except Exception as e: logger.error(f"[Quality Scanner] Critical error: {e}") diff --git a/core/discovery/sync.py b/core/discovery/sync.py index 3c602fb9..1f068450 100644 --- a/core/discovery/sync.py +++ b/core/discovery/sync.py @@ -39,8 +39,7 @@ class SyncDeps: """Bundle of cross-cutting deps the sync worker needs.""" config_manager: Any sync_service: Any - plex_client: Any - jellyfin_client: Any + media_server_engine: Any automation_engine: Any run_async: Callable[..., Any] record_sync_history_start: Callable @@ -227,8 +226,9 @@ def run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, p # Check sync service components logger.info(f" spotify_client: {sync_service.spotify_client is not None}") - logger.info(f" deps.plex_client: {sync_service.plex_client is not None}") - logger.info(f" deps.jellyfin_client: {sync_service.jellyfin_client is not None}") + _ms_engine = getattr(sync_service, '_engine', None) + logger.info(f" plex_client: {(_ms_engine.client('plex') if _ms_engine else None) is not None}") + logger.info(f" jellyfin_client: {(_ms_engine.client('jellyfin') if _ms_engine else None) is not None}") # Check media server connection before starting from config.settings import config_manager @@ -290,8 +290,8 @@ def run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, p logger.debug(f"Sync cache hit: '{original_title}' → server track {cached['server_track_id']}") return DatabaseTrackCached(db_track_check), cached['confidence'] logger.warning(f"Sync cache stale for '{original_title}' — track gone") - except Exception: - pass + except Exception as e: + logger.debug("sync match cache fast-path failed: %s", e) # --- End cache fast-path --- # Try each artist (same logic as original) @@ -322,8 +322,8 @@ def run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, p spotify_id, me.clean_title(original_title), me.clean_artist(artist_name), active_server, db_track.id, db_track.title, confidence ) - except Exception: - pass + except Exception as e: + logger.debug("save sync match cache failed: %s", e) # Create mock track object for playlist creation class DatabaseTrackMock: @@ -404,11 +404,12 @@ def run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, p try: active_server = deps.config_manager.get_active_media_server() logger.info(f"[PLAYLIST IMAGE] active_server={active_server}") - if active_server == 'plex' and deps.plex_client: - ok = deps.plex_client.set_playlist_image(playlist_name, playlist_image_url) + _engine = deps.media_server_engine + if active_server == 'plex' and _engine and _engine.client('plex'): + ok = _engine.client('plex').set_playlist_image(playlist_name, playlist_image_url) logger.info(f"[PLAYLIST IMAGE] Plex upload result: {ok}") - elif active_server in ('jellyfin', 'emby') and deps.jellyfin_client: - ok = deps.jellyfin_client.set_playlist_image(playlist_name, playlist_image_url) + elif active_server in ('jellyfin', 'emby') and _engine and _engine.client('jellyfin'): + ok = _engine.client('jellyfin').set_playlist_image(playlist_name, playlist_image_url) logger.info(f"[PLAYLIST IMAGE] Jellyfin upload result: {ok}") # Navidrome doesn't support custom playlist images except Exception as img_err: @@ -428,8 +429,8 @@ def run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, p entry = db.get_sync_history_entry(_resync_entry_id) if entry: target_batch_id = entry.get('batch_id', sync_batch_id) - except Exception: - pass + except Exception as e: + logger.debug("resync history lookup failed: %s", e) else: db.update_sync_history_completion(sync_batch_id, matched, synced, failed) @@ -465,8 +466,8 @@ def run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, p 'synced_tracks': str(getattr(result, 'synced_tracks', 0)), 'failed_tracks': str(getattr(result, 'failed_tracks', 0)), }) - except Exception: - pass + except Exception as e: + logger.debug("playlist_synced emit failed: %s", e) # Save sync status with match counts and track hash for smart-skip on next scheduled sync import hashlib as _hl diff --git a/core/download_engine/__init__.py b/core/download_engine/__init__.py new file mode 100644 index 00000000..f2ba73b0 --- /dev/null +++ b/core/download_engine/__init__.py @@ -0,0 +1,31 @@ +"""Download Engine — central owner of cross-source download state, +thread workers, search retry, rate-limits, and fallback chains. + +This is the second leg of the multi-source download dispatcher +refactor (the first leg, ``core/download_plugins/``, defined the +contract). The engine takes ownership of everything that used to +be duplicated across the per-source clients (background thread +workers, active_downloads dicts, search retry ladders, quality +filtering, hybrid fallback). Clients become DUMB — just hit the +API for their source, manage their own auth state, and let the +engine drive everything else. + +This package is built up in phases (see +``docs/download-engine-refactor-plan.md`` for the full plan): + +- Phase B (current) — engine skeleton + state lift. +- Phase C — background download worker. +- Phase D — search retry + quality filter. +- Phase E — rate-limit pool. +- Phase F — fallback chain. + +Each phase is purely additive at first (engine grows, clients +unchanged). Migration to the new shape happens one source per +commit so behavior never breaks across the suite. +""" + +from core.download_engine.engine import DownloadEngine +from core.download_engine.rate_limit import RateLimitPolicy +from core.download_engine.worker import BackgroundDownloadWorker + +__all__ = ["DownloadEngine", "BackgroundDownloadWorker", "RateLimitPolicy"] diff --git a/core/download_engine/engine.py b/core/download_engine/engine.py new file mode 100644 index 00000000..8aaa07e8 --- /dev/null +++ b/core/download_engine/engine.py @@ -0,0 +1,457 @@ +"""DownloadEngine — central owner of cross-source download state. + +Phase B scope: skeleton only. The engine exposes a place for +plugins to register, a single ``active_downloads`` dict keyed by +``(source, download_id)``, and per-source RLocks that guard mutations +without serializing workers across different sources. + +Subsequent phases bolt more capability on top: +- ``dispatch_download(plugin, target_id)`` (Phase C — replaces every + client's ``_download_thread_worker`` boilerplate). +- ``search(query, source_chain)`` (Phase D — replaces every client's + retry ladder + quality filter). +- ``rate_limit.acquire(source)`` (Phase E — replaces every client's + semaphore + last-download-timestamp dance). +- ``search_with_fallback`` / ``download_with_fallback`` (Phase F — + unifies hybrid mode across search and download). + +The engine is constructed by ``DownloadOrchestrator.__init__`` and +each plugin from the registry is registered with it. In Phase B +nothing in the existing code paths goes through the engine yet — +this commit is pure additive scaffolding so subsequent commits can +introduce engine-driven behavior one piece at a time without a +big-bang switchover. +""" + +from __future__ import annotations + +import threading +from typing import Any, Dict, Iterator, List, Optional, Tuple + +from utils.logging_config import get_logger + +logger = get_logger("download_engine") + + +# Type alias for the per-download state dict. Today's clients each +# define their own slightly-different shape (see Phase A pinning +# tests); the engine stores them as opaque dicts and the per-plugin +# accessor preserves the source-specific fields. +DownloadRecord = Dict[str, Any] + + +class DownloadEngine: + """Central state for every active download across every source. + + State is keyed by ``(source_name, download_id)`` so the same + UUID could hypothetically appear in two sources without + collision (in practice each source generates its own UUID4 + so collisions are negligible — the source qualifier exists + so the engine can answer "which plugin owns this download" in + O(1) without iterating every plugin). + + Thread safety: per-source lock sharding. Each source gets its own + RLock — progress callbacks on Deezer don't block Tidal's worker + and vice versa, matching the pre-refactor behavior where each + client owned its own download lock. Read-only accessors + (``get_record``, ``iter_records_for_source``) take the source's + lock briefly and return a SHALLOW COPY so the caller can iterate + without holding the lock. Callers that need to mutate a record + should use ``update_record`` which takes the lock and applies the + patch atomically. + """ + + def __init__(self) -> None: + # Nested dict: source_name → {download_id → record}. Replaces + # the original single-dict composite-key layout so + # ``iter_records_for_source`` is O(source_records) instead of + # O(total_records). + self._records: Dict[str, Dict[str, DownloadRecord]] = {} + # Per-source RLocks. Each source gets its own so progress + # updates on one source never block writes on another. RLock + # so a plugin's worker callback can re-enter while holding the + # lock for its own update. Lazily created via ``_source_lock``; + # the meta-lock guards creation against the create-race window + # where two threads could both miss + both create. + self._source_locks: Dict[str, threading.RLock] = {} + self._source_locks_lock = threading.Lock() + # Plugins that have registered with the engine. Source name + # → plugin instance. + self._plugins: Dict[str, Any] = {} + # Alias → canonical-name map. Lets engine resolve legacy + # source-name strings (e.g. ``'deezer_dl'`` for Deezer) to + # the canonical key in ``_plugins``. Cin's review caught + # that engine.cancel_download(source_hint='deezer_dl') + # silently fell through to Soulseek because alias resolution + # only existed at the registry, not on the engine. + self._aliases: Dict[str, str] = {} + # Background download worker — lives on the engine because + # it owns the cross-source state the worker mutates. Lazy + # import keeps the engine module standalone. + from core.download_engine.worker import BackgroundDownloadWorker + self.worker = BackgroundDownloadWorker(self) + + # ------------------------------------------------------------------ + # Plugin registration + # ------------------------------------------------------------------ + + def register_plugin(self, source_name: str, plugin: Any, + aliases: Tuple[str, ...] = ()) -> None: + """Register a plugin under its canonical source name. Called + once per source by the orchestrator after the registry's + ``initialize`` builds the client instances. + + ``aliases`` is the list of legacy source-name strings that + should resolve to this plugin (e.g. ``'deezer_dl'`` for + Deezer). Without alias resolution the engine couldn't route + cancel/lookup calls that came in with the legacy name. + + If the plugin exposes ``set_engine(engine)``, the engine + passes a self-reference so the plugin can dispatch into + ``engine.worker`` / read state / etc. Plugins that haven't + been migrated to the engine yet simply don't define + ``set_engine`` — they keep their pre-engine behavior + unchanged. + + Also reads the plugin's declared ``RateLimitPolicy`` (via + the ``rate_limit_policy()`` method or ``RATE_LIMIT_POLICY`` + class attribute) and applies it to the worker. Plugins that + don't declare a policy get the conservative default + (concurrency=1, delay=0). + """ + if source_name in self._plugins: + logger.warning("Plugin %s already registered with engine — overwriting", source_name) + self._plugins[source_name] = plugin + for alias in aliases: + self._aliases[alias] = source_name + + # Apply the plugin's rate-limit policy BEFORE set_engine so + # set_engine callbacks can override per-source if they need + # config-driven values (e.g. YouTube's user-tunable delay). + from core.download_engine.rate_limit import resolve_policy + policy = resolve_policy(plugin) + self.worker.set_concurrency(source_name, policy.download_concurrency) + self.worker.set_delay(source_name, policy.download_delay_seconds) + + set_engine = getattr(plugin, 'set_engine', None) + if callable(set_engine): + try: + set_engine(self) + except Exception as exc: + logger.warning( + "Plugin %s set_engine callback failed: %s", source_name, exc, + ) + + def get_plugin(self, source_name: str) -> Optional[Any]: + """Return the plugin instance for the given source name. + Resolves through aliases — e.g. ``get_plugin('deezer_dl')`` + returns the same instance as ``get_plugin('deezer')``.""" + if source_name in self._plugins: + return self._plugins[source_name] + canonical = self._aliases.get(source_name) + if canonical: + return self._plugins.get(canonical) + return None + + def _resolve_canonical(self, source_name: str) -> Optional[str]: + """Return the canonical source name for an input that may be + an alias. Returns None if the input matches neither a + canonical name nor an alias.""" + if source_name in self._plugins: + return source_name + return self._aliases.get(source_name) + + def registered_sources(self) -> List[str]: + return list(self._plugins.keys()) + + def _source_lock(self, source_name: str) -> threading.RLock: + """Return the per-source RLock, lazy-creating it on first use. + The meta-lock around the cache lookup closes the create-race + window where two threads both miss + both create a fresh lock. + """ + with self._source_locks_lock: + lock = self._source_locks.get(source_name) + if lock is None: + lock = threading.RLock() + self._source_locks[source_name] = lock + return lock + + # ------------------------------------------------------------------ + # Active-downloads state — Phase B core surface + # ------------------------------------------------------------------ + + def add_record(self, source_name: str, download_id: str, record: DownloadRecord) -> None: + """Insert a fresh download record. Used by clients (today + directly via their own dicts; Phase B2 routes them through + here).""" + with self._source_lock(source_name): + source_bucket = self._records.setdefault(source_name, {}) + if download_id in source_bucket: + logger.warning("Replacing existing download record for %s/%s", source_name, download_id) + source_bucket[download_id] = dict(record) + + def update_record(self, source_name: str, download_id: str, patch: DownloadRecord) -> None: + """Apply a partial patch to an existing record. No-op if the + record was already removed (e.g. cancelled mid-update).""" + with self._source_lock(source_name): + existing = self._records.get(source_name, {}).get(download_id) + if existing is None: + return + existing.update(patch) + + def update_record_unless_state(self, source_name: str, download_id: str, + patch: DownloadRecord, + skip_if_state_in: Tuple[str, ...] = ()) -> bool: + """Atomically check the record's state and apply ``patch`` only + if the current state is NOT in ``skip_if_state_in``. Returns + True if the patch was applied, False if it was skipped (or + the record didn't exist). + + Used by the background download worker's ``_mark_terminal`` + to avoid the read-then-write race Cin flagged: a cancel + landing between the snapshot and update could be overwritten + back to Errored / Completed. Holding the source's lock across + the check + write closes the window. + """ + with self._source_lock(source_name): + existing = self._records.get(source_name, {}).get(download_id) + if existing is None: + return False + if existing.get('state') in skip_if_state_in: + return False + existing.update(patch) + return True + + def remove_record(self, source_name: str, download_id: str) -> Optional[DownloadRecord]: + """Delete a record (cancellation cleanup). Returns the + removed record or None if not found.""" + with self._source_lock(source_name): + source_bucket = self._records.get(source_name) + if not source_bucket: + return None + removed = source_bucket.pop(download_id, None) + # Drop the empty source bucket so iteration / membership + # checks don't see a stale source key. + if not source_bucket: + self._records.pop(source_name, None) + return removed + + def get_record(self, source_name: str, download_id: str) -> Optional[DownloadRecord]: + """Return a SHALLOW COPY of the record. Caller mutations + don't affect engine state — use ``update_record`` for that.""" + with self._source_lock(source_name): + record = self._records.get(source_name, {}).get(download_id) + return dict(record) if record is not None else None + + def iter_records_for_source(self, source_name: str) -> Iterator[DownloadRecord]: + """Yield SHALLOW COPIES of every record owned by a source. + Holds the source's lock briefly to snapshot, then yields + outside the lock so callers can spend arbitrary time on each + record. + + With the nested-dict layout this is O(source_records) — only + touches the bucket for the requested source, not every record + across every source. + """ + with self._source_lock(source_name): + source_bucket = self._records.get(source_name, {}) + snapshot = [dict(record) for record in source_bucket.values()] + for record in snapshot: + yield record + + # ------------------------------------------------------------------ + # Cross-source query dispatch — Phase B2 surface + # ------------------------------------------------------------------ + # + # The orchestrator historically iterated every plugin in its own + # ``get_all_downloads`` / ``get_download_status`` / ``cancel_download`` + # methods (with hand-maintained client lists, before the registry + # came along). That iteration logic moves into the engine here so + # the orchestrator becomes a thin pass-through (Phase B3). + # + # In Phase B these methods iterate the registered plugins and call + # their existing ``get_all_downloads`` / ``cancel_download`` + # methods — same behavior as today, just in a new home. Phase C/D + # will replace plugin-iteration with direct engine-state queries + # once the thread worker is also lifted. + # + # All methods are async to match the per-plugin contract. + + async def get_all_downloads(self, exclude: Tuple[str, ...] = ()): + """Aggregated view across every registered plugin's active + downloads. Per-plugin exceptions are swallowed (one source + failing shouldn't take down cross-source aggregation) but + logged at debug level — same defensive shape the legacy + orchestrator had. + + ``exclude`` skips named sources entirely. The download monitor + passes ``('soulseek',)`` so it doesn't double-fetch slskd + transfers (it already pulled them via the slskd transfers + endpoint earlier in the same loop). + """ + all_downloads = [] + for source_name, plugin in self._plugins.items(): + if plugin is None or source_name in exclude: + continue + try: + all_downloads.extend(await plugin.get_all_downloads()) + except Exception as exc: + logger.debug("%s get_all_downloads failed: %s", source_name, exc) + return all_downloads + + async def get_download_status(self, download_id: str): + """Find a download_id across every plugin. Returns the first + plugin's response or None if no plugin owns it.""" + for source_name, plugin in self._plugins.items(): + if plugin is None: + continue + try: + status = await plugin.get_download_status(download_id) + if status: + return status + except Exception as exc: + logger.debug("%s get_download_status failed: %s", source_name, exc) + return None + + async def cancel_download(self, download_id: str, + source_hint: Optional[str] = None, + remove: bool = False) -> bool: + """Cancel a download. ``source_hint`` is the source name (or + legacy alias like ``'deezer_dl'``, or a real Soulseek peer + username) — when provided, routes directly to that plugin. + When omitted, every plugin is asked in turn until one accepts. + + Cin's review caught a bug here: legacy alias strings like + ``'deezer_dl'`` weren't resolved to the canonical ``'deezer'`` + plugin name, so the cancel silently fell through to Soulseek. + Resolution now goes through ``_resolve_canonical`` first. + """ + # Direct routing when the caller knows the source. + if source_hint: + canonical = self._resolve_canonical(source_hint) + # Streaming source names (or aliases) resolve to a + # registered plugin. Anything else (real Soulseek peer + # name not in our registry) routes to Soulseek. + if canonical and canonical != 'soulseek': + target_plugin = self._plugins.get(canonical) + if target_plugin is not None: + try: + return await target_plugin.cancel_download( + download_id, source_hint, remove, + ) + except Exception as exc: + logger.debug("%s cancel_download failed: %s", canonical, exc) + return False + soulseek = self._plugins.get('soulseek') + if soulseek is not None: + try: + return await soulseek.cancel_download(download_id, source_hint, remove) + except Exception as exc: + logger.debug("soulseek cancel_download failed: %s", exc) + return False + + # No hint → ask every plugin until one cancels successfully. + for source_name, plugin in self._plugins.items(): + if plugin is None: + continue + try: + if await plugin.cancel_download(download_id, source_hint, remove): + return True + except Exception as exc: + logger.debug("%s cancel_download failed: %s", source_name, exc) + return False + + async def clear_all_completed_downloads(self) -> bool: + """Best-effort cleanup of every plugin's completed-downloads + list. Skips plugins that report not-configured (saves API + calls + log noise).""" + results = [] + for source_name, plugin in self._plugins.items(): + if plugin is None: + continue + if hasattr(plugin, 'is_configured') and not plugin.is_configured(): + logger.debug("Skipping %s clear_all_completed_downloads (not configured)", source_name) + continue + try: + results.append(await plugin.clear_all_completed_downloads()) + except Exception as exc: + logger.warning("%s clear_all_completed_downloads failed: %s", source_name, exc) + results.append(False) + return all(results) if results else True + + # ------------------------------------------------------------------ + # Hybrid fallback — Phase F surface + # ------------------------------------------------------------------ + + async def search_with_fallback(self, query: str, source_chain, + timeout=None, progress_callback=None): + """Try each source in ``source_chain`` until one returns + tracks. Skips unconfigured / unregistered sources, swallows + per-source exceptions. Returns the first non-empty + (tracks, albums) tuple, or ``([], [])`` when every source + in the chain is exhausted. + + Replaces orchestrator's hand-rolled hybrid search loop. The + chain is ordered (most-preferred first). + """ + for i, source_name in enumerate(source_chain): + plugin = self._plugins.get(source_name) + if plugin is None: + logger.info(f"Skipping {source_name} (not available)") + continue + if hasattr(plugin, 'is_configured') and not plugin.is_configured(): + logger.info(f"Skipping {source_name} (not configured)") + continue + + try: + logger.info(f"Trying {source_name} (priority {i+1}): {query}") + tracks, albums = await plugin.search(query, timeout, progress_callback) + if tracks: + logger.info(f"{source_name} found {len(tracks)} tracks") + return (tracks, albums) + except Exception as e: + logger.warning(f"{source_name} search failed: {e}") + + logger.warning( + "Hybrid search: all sources (%s) found nothing for: %s", + ', '.join(source_chain), query, + ) + return ([], []) + + async def download_with_fallback(self, username: str, filename: str, + file_size: int, source_chain) -> Optional[str]: + """Try each source in ``source_chain`` until one accepts the + download (returns a non-None download_id). Fixes the legacy + bug where hybrid mode silently routed to a single source via + the username hint with no retry on failure. + + ``username`` is treated as a hint when it matches a source + name in the chain — that source is tried FIRST regardless of + chain order. Anything else (e.g. a real Soulseek peer name) + routes through the chain in declared order. + """ + # Promote a matching source-name hint to the head of the chain. + ordered_chain = list(source_chain) + if username and username in ordered_chain: + ordered_chain.remove(username) + ordered_chain.insert(0, username) + + for source_name in ordered_chain: + plugin = self._plugins.get(source_name) + if plugin is None: + continue + if hasattr(plugin, 'is_configured') and not plugin.is_configured(): + continue + try: + download_id = await plugin.download(username, filename, file_size) + if download_id is not None: + return download_id + logger.info(f"{source_name} declined download — trying next in chain") + except Exception as e: + logger.warning(f"{source_name} download raised — trying next in chain: {e}") + + logger.warning( + "Hybrid download: every source in chain (%s) refused %r", + ', '.join(ordered_chain), filename, + ) + return None diff --git a/core/download_engine/rate_limit.py b/core/download_engine/rate_limit.py new file mode 100644 index 00000000..65d33c86 --- /dev/null +++ b/core/download_engine/rate_limit.py @@ -0,0 +1,76 @@ +"""Per-source rate-limit policy declarations. + +Today's per-source download throttling is scattered: + +- YouTube: ``self._download_delay = config_manager.get('youtube.download_delay', 3)`` + set in ``__init__``, applied in ``set_engine`` via worker.set_delay. +- Qobuz: module-level ``_qobuz_api_lock`` + ``_QOBUZ_MIN_INTERVAL`` for + search-side throttling, no download-side throttle. +- Other sources: no explicit declarations — default to 0s delay / + concurrency=1, which works because the streaming APIs have their + own gateway-level rate limits. + +Phase E centralizes this into one place: each plugin declares a +``RateLimitPolicy`` (either as a class attribute or returned from a +``rate_limit_policy()`` method), and the engine reads + applies the +policy to ``engine.worker`` at ``register_plugin`` time. + +Adding a new source = declaring its policy alongside the rest of +the source's auth/config — no longer a hidden line in __init__ or a +module-level constant in the client file. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class RateLimitPolicy: + """Per-source download throttling policy. + + Attributes: + download_concurrency: Max number of concurrent downloads + from this source. Default 1 (serial). Most streaming + APIs prefer serial transfers because parallel just + trades rate-limit errors for thread overhead. + download_delay_seconds: Minimum gap between successive + downloads from this source. YouTube uses 3s today + (legacy ``_download_delay`` config key) to avoid + yt-dlp 429s. Most other sources use 0. + """ + + download_concurrency: int = 1 + download_delay_seconds: float = 0.0 + + +# Sentinel default — most plugins want this. Plugins that need +# tighter throttling override by exposing ``RATE_LIMIT_POLICY`` as +# a class attribute or returning a custom one from +# ``rate_limit_policy()``. +DEFAULT_POLICY = RateLimitPolicy() + + +def resolve_policy(plugin) -> RateLimitPolicy: + """Read a plugin's declared rate-limit policy. Checks (in order): + 1. ``plugin.rate_limit_policy()`` method (returns a RateLimitPolicy) + 2. ``plugin.RATE_LIMIT_POLICY`` class attribute + 3. ``DEFAULT_POLICY`` + """ + method = getattr(plugin, 'rate_limit_policy', None) + if callable(method): + try: + policy = method() + if isinstance(policy, RateLimitPolicy): + return policy + except Exception as e: + logger.debug("plugin rate_limit_policy() call failed: %s", e) + + declared = getattr(plugin, 'RATE_LIMIT_POLICY', None) + if isinstance(declared, RateLimitPolicy): + return declared + + return DEFAULT_POLICY diff --git a/core/download_engine/worker.py b/core/download_engine/worker.py new file mode 100644 index 00000000..54f36757 --- /dev/null +++ b/core/download_engine/worker.py @@ -0,0 +1,315 @@ +"""BackgroundDownloadWorker — engine-owned thread spawning + state +lifecycle for downloads. + +Today every streaming download client (YouTube, Tidal, Qobuz, HiFi, +Deezer, SoundCloud) hand-rolls the same thread-spawn pattern: + +```python +async def download(self, ...): + download_id = str(uuid.uuid4()) + with self._download_lock: + self.active_downloads[download_id] = {...initial state...} + threading.Thread( + target=self._download_thread_worker, + args=(download_id, target_id, display_name, ...), + daemon=True, + ).start() + return download_id + +def _download_thread_worker(self, download_id, target_id, display_name, ...): + with self._download_semaphore: + # rate-limit sleep + # update state to 'InProgress, Downloading' + file_path = self._download_sync(...) # the source-specific atomic op + # update state to 'Completed, Succeeded' / 'Errored' +``` + +That pattern is duplicated 6+ times across the codebase (~70 LOC +each, ~490 total). The worker class lifts it into the engine — each +plugin only has to provide the atomic op (``impl_callable``) and +declare its rate-limit policy. Adding a new download source becomes +a much smaller patch. + +Phase C1 scope: introduce the worker. No client migrated yet — the +worker just exists for C2–C7 to migrate sources one at a time, each +under a passing pinning test. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from typing import Any, Callable, Dict, Optional + +from utils.logging_config import get_logger + +logger = get_logger("download_engine.worker") + + +# Type aliases for clarity. ``ImplCallable`` is the per-plugin +# atomic download operation — synchronous, returns a file path on +# success or raises (or returns None) on failure. +ImplCallable = Callable[[str, Any, str], Optional[str]] + + +class BackgroundDownloadWorker: + """Engine-owned thread spawner for per-source downloads. + + State-machine semantics (preserved verbatim from the legacy + per-client workers so consumers reading these fields keep + working): + + - ``Initializing`` — set on dispatch, before the thread starts. + - ``InProgress, Downloading`` — set when the worker thread + acquires the semaphore and is about to call the impl. + - ``Completed, Succeeded`` — set when impl returns a non-None + file path. ``progress=100.0`` and ``file_path=`` + also written. + - ``Errored`` — set when impl returns None OR raises. The + record is left in place so downstream consumers can inspect + what failed. + + Per-source serialization: each source gets a ``threading.Semaphore`` + (default size 1, configurable per-source via ``set_concurrency``). + Same shape the existing clients use today (each source defines + its own semaphore). Engine owning them centrally lets a future + Phase E rate-limiter swap the semaphore for a smarter pool. + + Per-source delay-between-downloads: default 0 seconds (most + sources don't need it). YouTube currently uses 3s, Qobuz uses + 1s — the legacy values get configured in via ``set_delay`` + when the source registers. + """ + + def __init__(self, engine: Any) -> None: + self._engine = engine + # Per-source semaphores + delay state. The first dispatch + # for a source auto-creates a semaphore with concurrency=1 + # if the source hasn't been configured explicitly. + self._semaphores: Dict[str, threading.Semaphore] = {} + self._delays: Dict[str, float] = {} + self._last_download_at: Dict[str, float] = {} + self._config_lock = threading.Lock() + + # ------------------------------------------------------------------ + # Per-source rate-limit configuration + # ------------------------------------------------------------------ + + def set_concurrency(self, source_name: str, max_concurrent: int) -> None: + """Set the max number of concurrent downloads for a source. + Default is 1 (serial). Most sources will keep the default — + the streaming APIs all rate-limit at the API gateway level + anyway, parallel downloads just trade rate-limit errors for + thread overhead.""" + with self._config_lock: + self._semaphores[source_name] = threading.Semaphore(max_concurrent) + + def set_delay(self, source_name: str, seconds: float) -> None: + """Set a minimum delay between successive downloads from the + same source. YouTube uses 3s today (avoid yt-dlp 429s), + Qobuz uses 1s. Other sources use 0 (no delay).""" + with self._config_lock: + self._delays[source_name] = float(seconds) + + def _get_semaphore(self, source_name: str) -> threading.Semaphore: + with self._config_lock: + sem = self._semaphores.get(source_name) + if sem is None: + sem = threading.Semaphore(1) + self._semaphores[source_name] = sem + return sem + + def _get_delay(self, source_name: str) -> float: + with self._config_lock: + return self._delays.get(source_name, 0.0) + + # ------------------------------------------------------------------ + # Dispatch — public API + # ------------------------------------------------------------------ + + def dispatch( + self, + source_name: str, + target_id: Any, + display_name: str, + original_filename: str, + impl_callable: ImplCallable, + extra_record_fields: Optional[Dict[str, Any]] = None, + username_override: Optional[str] = None, + thread_name: Optional[str] = None, + ) -> str: + """Kick off a background download. + + Args: + source_name: Canonical source name (e.g. 'youtube', + 'tidal'). Used as the engine state key + the + username slot in the record (unless overridden). + target_id: Source-specific identifier (track_id, video_id, + permalink_url, album_foreign_id, etc.). Passed + verbatim to ``impl_callable``. + display_name: Human-readable label for logs / UI. + original_filename: The encoded filename the orchestrator + received (e.g. ``'12345||Song Title'``). Stored in + the record's ``filename`` slot for context-key lookups. + impl_callable: Synchronous function that performs the + actual download. Signature: + ``impl_callable(download_id, target_id, display_name) -> Optional[str]``. + Returns the final file path on success or None / + raises on failure. + extra_record_fields: Per-source extras to merge into the + initial record (e.g. ``{'video_id': '...', 'url': + '...', 'title': '...'}`` for YouTube). Used to + preserve source-specific slots that downstream + consumers + status APIs read. + username_override: Use this instead of ``source_name`` + in the record's ``username`` slot. Required for + Deezer (legacy ``'deezer_dl'``) — every other source + uses the canonical name. + thread_name: Optional thread name for diagnostics. Deezer + uses ``'deezer-dl-'`` — Phase A pinning + tests catch any drift in this convention. + + Returns: + download_id (UUID4 string). The orchestrator polls via + ``engine.get_download_status(download_id)`` for progress. + """ + download_id = str(uuid.uuid4()) + + record: Dict[str, Any] = { + 'id': download_id, + 'filename': original_filename, + 'username': username_override or source_name, + 'state': 'Initializing', + 'progress': 0.0, + 'size': 0, + 'transferred': 0, + 'speed': 0, + 'time_remaining': None, + 'file_path': None, + } + if extra_record_fields: + record.update(extra_record_fields) + + self._engine.add_record(source_name, download_id, record) + + thread = threading.Thread( + target=self._worker_loop, + args=(source_name, download_id, target_id, display_name, impl_callable), + daemon=True, + name=thread_name, + ) + thread.start() + + return download_id + + # ------------------------------------------------------------------ + # Worker thread — the lifted boilerplate + # ------------------------------------------------------------------ + + def _worker_loop( + self, + source_name: str, + download_id: str, + target_id: Any, + display_name: str, + impl_callable: ImplCallable, + ) -> None: + """Runs on the spawned daemon thread. Handles semaphore + acquisition, rate-limit sleep, state lifecycle, exception + capture. The plugin-specific work happens entirely inside + ``impl_callable``.""" + try: + with self._get_semaphore(source_name): + # Rate-limit delay against the LAST download from + # this source (not just this worker — semaphore + # ensures serial access while delay is configured). + delay = self._get_delay(source_name) + if delay > 0: + last_at = self._last_download_at.get(source_name, 0.0) + elapsed = time.time() - last_at + if last_at > 0 and elapsed < delay: + wait_time = delay - elapsed + logger.info( + "Rate-limit delay for %s: waiting %.1fs before next download", + source_name, wait_time, + ) + time.sleep(wait_time) + + self._engine.update_record(source_name, download_id, { + 'state': 'InProgress, Downloading', + }) + + try: + file_path = impl_callable(download_id, target_id, display_name) + except Exception as exc: + logger.error( + "%s download %s failed (impl raised): %s", + source_name, download_id, exc, + ) + self._mark_terminal( + source_name, download_id, + success=False, error=str(exc), + ) + return + + self._last_download_at[source_name] = time.time() + + if file_path: + # Atomic write — preserve Cancelled if user cancelled + # between impl returning and this write. Same guard + # _mark_terminal uses; Cin flagged both split sites. + self._engine.update_record_unless_state( + source_name, download_id, + { + 'state': 'Completed, Succeeded', + 'progress': 100.0, + 'file_path': file_path, + }, + skip_if_state_in=('Cancelled',), + ) + logger.info( + "%s download %s completed: %s", + source_name, download_id, file_path, + ) + else: + self._mark_terminal(source_name, download_id, success=False) + logger.error( + "%s download %s failed (impl returned None)", + source_name, download_id, + ) + + except Exception as exc: + # Defensive — semaphore / sleep shouldn't blow up the + # thread, but if they do the record needs SOME terminal + # state or it sits at 'Initializing' forever. + logger.exception( + "%s worker_loop crashed for download %s: %s", + source_name, download_id, exc, + ) + self._mark_terminal( + source_name, download_id, + success=False, error=f'worker crash: {exc}', + ) + + def _mark_terminal(self, source_name: str, download_id: str, + success: bool, error: Optional[str] = None) -> None: + """Write a terminal state, but DON'T clobber an explicit + 'Cancelled' state set by the user via cancel_download. + Mirrors the legacy per-client guard + (``if state != 'Cancelled': state = 'Errored'``) every + client used to hand-roll inside its thread worker. + + Uses ``update_record_unless_state`` so the check + write are + atomic under the engine's per-source lock. Cin caught a race + where a cancel landing between the read-snapshot + write + could overwrite Cancelled back to Errored / Completed. + """ + patch: Dict[str, Any] = { + 'state': 'Completed, Succeeded' if success else 'Errored', + } + if error is not None: + patch['error'] = error + self._engine.update_record_unless_state( + source_name, download_id, patch, skip_if_state_in=('Cancelled',), + ) diff --git a/core/download_orchestrator.py b/core/download_orchestrator.py index 41abee9b..e8231e09 100644 --- a/core/download_orchestrator.py +++ b/core/download_orchestrator.py @@ -1,15 +1,22 @@ """ Download Orchestrator -Routes downloads between Soulseek, YouTube, Tidal, Qobuz, HiFi, and Deezer based on configuration. +Routes downloads between Soulseek, YouTube, Tidal, Qobuz, HiFi, Deezer, and SoundCloud based on configuration. -Supports seven modes: +Supports eight modes: - Soulseek Only: Traditional behavior - YouTube Only: YouTube-exclusive downloads - Tidal Only: Tidal-exclusive downloads - Qobuz Only: Qobuz-exclusive downloads - HiFi Only: Free lossless downloads via public hifi-api instances - Deezer Only: Deezer downloads via ARL authentication +- SoundCloud Only: Anonymous SoundCloud downloads (DJ mixes, removed/exclusive tracks) - Hybrid: Try primary source first, fallback to others + +The orchestrator dispatches through ``core.download_plugins.registry`` +instead of hardcoded per-source ``[self.soulseek, self.youtube, ...]`` +lists. External callers reach individual clients via the generic +``orchestrator.client('')`` accessor (alias-aware), not direct +attribute access. """ import asyncio @@ -18,13 +25,9 @@ from pathlib import Path from utils.logging_config import get_logger from config.settings import config_manager -from core.soulseek_client import SoulseekClient, TrackResult, AlbumResult, DownloadStatus -from core.youtube_client import YouTubeClient -from core.tidal_download_client import TidalDownloadClient -from core.qobuz_client import QobuzClient -from core.hifi_client import HiFiClient -from core.deezer_download_client import DeezerDownloadClient -from core.lidarr_download_client import LidarrDownloadClient +from core.download_engine import DownloadEngine +from core.download_plugins.registry import DownloadPluginRegistry, build_default_registry +from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus logger = get_logger("download_orchestrator") @@ -37,18 +40,31 @@ class DownloadOrchestrator: Routes requests to the appropriate client(s) based on configured mode. """ - def __init__(self): - """Initialize orchestrator with all clients. - Each client is initialized independently — one failing client doesn't prevent others from working.""" - self._init_failures = [] + def __init__(self, registry: Optional[DownloadPluginRegistry] = None, + engine: Optional[DownloadEngine] = None): + """Initialize orchestrator with a plugin registry. Each plugin + is built and registered independently — one failing plugin + doesn't prevent others from working. The ``registry`` arg + exists so tests can inject a registry with mock plugins; in + production callers leave it None and get the default. - self.soulseek = self._safe_init('Soulseek', SoulseekClient) - self.youtube = self._safe_init('YouTube', YouTubeClient) - self.tidal = self._safe_init('Tidal', TidalDownloadClient) - self.qobuz = self._safe_init('Qobuz', QobuzClient) - self.hifi = self._safe_init('HiFi', HiFiClient) - self.deezer_dl = self._safe_init('Deezer', DeezerDownloadClient) - self.lidarr = self._safe_init('Lidarr', LidarrDownloadClient) + ``engine`` is the cross-source state owner. Phase B introduces + it as a held reference; it isn't on any code path yet — Phase + C/D/E/F migrate behavior into it incrementally. + """ + self.registry = registry if registry is not None else build_default_registry() + self.registry.initialize() + self._init_failures = self.registry.init_failures + + # Engine — owns cross-source state, threading, search retry, + # rate-limits, fallback. Built in subsequent phases. For Phase + # B it's just an empty registry of plugins so future phases + # can route through it without further orchestrator changes. + self.engine = engine if engine is not None else DownloadEngine() + for source_name, plugin in self.registry.all_plugins(): + spec = self.registry.get_spec(source_name) + aliases = spec.aliases if spec else () + self.engine.register_plugin(source_name, plugin, aliases=aliases) if self._init_failures: logger.warning(f"Download clients failed to initialize: {', '.join(self._init_failures)}") @@ -68,15 +84,6 @@ class DownloadOrchestrator: self.hybrid_secondary, ) - def _safe_init(self, name, cls): - """Initialize a download client, returning None on failure instead of crashing.""" - try: - return cls() - except Exception as e: - logger.error(f"{name} download client failed to initialize: {e}") - self._init_failures.append(name) - return None - def reload_settings(self): """Reload settings from config (call after settings change)""" self.mode = config_manager.get('download_source.mode', 'soulseek') @@ -85,20 +92,28 @@ class DownloadOrchestrator: self.hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) # Reload underlying client configs (SLSKD URL, API key, etc.) - if self.soulseek: - self.soulseek._setup_client() + soulseek = self.client('soulseek') + if soulseek: + soulseek._setup_client() logger.info("Soulseek client config reloaded") # Reconnect Deezer if ARL changed deezer_arl = config_manager.get('deezer_download.arl', '') - if deezer_arl and self.deezer_dl: - self.deezer_dl.reconnect(deezer_arl) - self.deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac') + deezer_dl = self.client('deezer_dl') + if deezer_arl and deezer_dl: + deezer_dl.reconnect(deezer_arl) + deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac') - # Reload download path for all clients that cache it + # Reload download path for all clients that cache it. + # Soulseek owns the path config and is reloaded above; every + # other source mirrors that path so files all land in one + # tree. Sources without a `download_path` attribute (e.g. + # Lidarr — pulls into Lidarr's own tree) silently skip. new_path = Path(config_manager.get('soulseek.download_path', './downloads')) - for client in [self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl]: - if client and hasattr(client, 'download_path') and client.download_path != new_path: + for name, client in self.registry.all_plugins(): + if name == 'soulseek': + continue + if hasattr(client, 'download_path') and client.download_path != new_path: client.download_path = new_path client.download_path.mkdir(parents=True, exist_ok=True) # YouTube also caches path in yt-dlp opts @@ -108,11 +123,61 @@ class DownloadOrchestrator: logger.info(f"Download Orchestrator settings reloaded - Mode: {self.mode}") - def _client(self, name): - """Get a client by name, returning None if not initialized.""" - return {'soulseek': self.soulseek, 'youtube': self.youtube, 'tidal': self.tidal, - 'qobuz': self.qobuz, 'hifi': self.hifi, 'deezer_dl': self.deezer_dl, - 'lidarr': self.lidarr}.get(name) + def client(self, name): + """Generic accessor for a download source client by name. + + Cin's review feedback: external callers should reach into + per-source clients via this method (``orch.client('hifi')``) + instead of attribute access (``orch.hifi``). Resolves both + canonical names (``deezer``) and legacy aliases (``deezer_dl``) + via the registry. Returns None if the source isn't registered + or failed to initialize. + """ + return self.registry.get(name) + + # Internal alias kept for legacy callers inside this file. + _client = client + + def configured_clients(self) -> dict: + """Return ``{source_name: client}`` for every download source + that's both initialized AND reports is_configured() == True. + + Replaces the legacy per-source iteration pattern Cin called + out — `if hasattr(orch, 'soulseek') and orch.soulseek and + orch.soulseek.is_configured(): download_clients['soulseek'] + = orch.soulseek` repeated for each source. + """ + result = {} + for name, client in self.registry.all_plugins(): + try: + if not hasattr(client, 'is_configured') or client.is_configured(): + result[name] = client + except Exception as exc: + logger.debug("%s is_configured raised: %s", name, exc) + return result + + def reload_instances(self, source: str = None) -> bool: + """Reload a source's instance config (e.g. HiFi instance list, + Qobuz session restore). Generic dispatch — caller passes the + source name instead of reaching for ``orch.hifi.reload_instances()``. + + When ``source`` is None, reloads every source that has a + ``reload_instances`` method. + """ + sources = [source] if source else list(self.registry.names()) + ok = True + for name in sources: + client = self.client(name) + if client is None: + continue + if not hasattr(client, 'reload_instances'): + continue + try: + client.reload_instances() + except Exception as exc: + logger.warning("%s reload_instances failed: %s", name, exc) + ok = False + return ok def is_configured(self) -> bool: """ @@ -129,12 +194,18 @@ class DownloadOrchestrator: return False def get_source_status(self) -> dict: - """Return configured status for each download source.""" - return {name: (c.is_configured() if c else False) - for name, c in [('soulseek', self.soulseek), ('youtube', self.youtube), - ('tidal', self.tidal), ('qobuz', self.qobuz), - ('hifi', self.hifi), ('deezer_dl', self.deezer_dl), - ('lidarr', self.lidarr)]} + """Return configured status for each download source. + + Keys preserve the legacy ``deezer_dl`` alias used by the + frontend status indicators and per-source dispatch strings, + so callers reading specific keys keep working unchanged. + """ + status = {} + for name in self.registry.names(): + client = self.registry.get(name) + key = 'deezer_dl' if name == 'deezer' else name + status[key] = client.is_configured() if client else False + return status async def check_connection(self) -> bool: """ @@ -146,7 +217,7 @@ class DownloadOrchestrator: if client and self.mode != 'hybrid': return await client.check_connection() elif self.mode == 'hybrid': - sources_to_check = self.hybrid_order if self.hybrid_order else ['soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'] + sources_to_check = self.hybrid_order if self.hybrid_order else self.registry.names() results = {} for source in sources_to_check: client = self._client(source) @@ -165,76 +236,62 @@ class DownloadOrchestrator: return False + def _normalize_source_name(self, name: str) -> Optional[str]: + """Convert a possibly-aliased source name (e.g. legacy + ``'deezer_dl'``) to the canonical registry name (``'deezer'``). + Returns None if the input matches neither a canonical name + nor an alias. + + Cin's review caught a bug where legacy alias values from + config (hybrid_order containing ``'deezer_dl'``) silently + dropped Deezer from hybrid mode because the canonical-name + membership check rejected the alias. + """ + spec = self.registry.get_spec(name) if name else None + return spec.name if spec else None + + def _resolve_source_chain(self) -> List[str]: + """Order the configured sources for hybrid mode. Prefers + ``hybrid_order`` config; falls back to legacy + primary/secondary pair when no order set. Normalizes alias + names through the registry so legacy ``deezer_dl`` config + values resolve correctly to the canonical ``deezer`` plugin.""" + if self.hybrid_order: + chain = [] + seen = set() + for raw in self.hybrid_order: + canonical = self._normalize_source_name(raw) + if canonical and canonical not in seen: + chain.append(canonical) + seen.add(canonical) + return chain + primary = self._normalize_source_name(self.hybrid_primary) or 'soulseek' + secondary = self._normalize_source_name(self.hybrid_secondary) or 'soulseek' + if secondary == primary: + secondary = next( + (name for name in self.registry.names() if name != primary), + 'soulseek', + ) + chain = [primary, secondary] + if not chain: + chain = ['soulseek'] + return chain + async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]: - """ - Search for tracks using configured source(s). - - Args: - query: Search query - timeout: Search timeout (for Soulseek) - progress_callback: Progress callback (for Soulseek) - - Returns: - Tuple of (track_results, album_results) - """ - source_names = {'soulseek': 'Soulseek', 'youtube': 'YouTube', 'tidal': 'Tidal', - 'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr'} - + """Search for tracks using configured source(s). Single-source + modes route directly; hybrid mode delegates to + ``engine.search_with_fallback`` which tries the chain in order.""" if self.mode != 'hybrid': client = self._client(self.mode) if not client: - logger.error(f"{source_names.get(self.mode, self.mode)} client not available (failed to initialize)") + logger.error(f"{self.registry.display_name(self.mode)} client not available (failed to initialize)") return [], [] - logger.info(f"Searching {source_names.get(self.mode, self.mode)}: {query}") + logger.info(f"Searching {self.registry.display_name(self.mode)}: {query}") return await client.search(query, timeout, progress_callback) - elif self.mode == 'hybrid': - clients = {name: self._client(name) for name in source_names} - - # Build ordered source list: prefer hybrid_order, fall back to legacy primary/secondary - if self.hybrid_order: - source_order = [s for s in self.hybrid_order if s in clients] - else: - primary = self.hybrid_primary if self.hybrid_primary in clients else 'soulseek' - secondary = self.hybrid_secondary if self.hybrid_secondary in clients else 'soulseek' - if secondary == primary: - secondary = next((name for name in clients if name != primary), 'soulseek') - source_order = [primary, secondary] - - if not source_order: - source_order = ['soulseek'] - - logger.info(f"Hybrid search ({' → '.join(source_order)}): {query}") - - # Try each source in priority order (skip unconfigured/unavailable ones) - for i, source_name in enumerate(source_order): - client = clients.get(source_name) - if not client: - logger.info(f"Skipping {source_name} (not available)") - continue - if hasattr(client, 'is_configured') and not client.is_configured(): - logger.info(f"Skipping {source_name} (not configured)") - continue - - try: - if i == 0: - logger.info(f"Trying {source_name} (priority {i+1}): {query}") - else: - logger.info(f"Trying {source_name} (priority {i+1}): {query}") - - tracks, albums = await client.search(query, timeout, progress_callback) - if tracks: - logger.info(f"{source_name} found {len(tracks)} tracks") - return (tracks, albums) - except Exception as e: - logger.warning(f"{source_name} search failed: {e}") - - # Nothing found from any source - logger.warning(f"Hybrid search: all sources ({', '.join(source_order)}) found nothing for: {query}") - return ([], []) - - # Fallback: empty results - return ([], []) + chain = self._resolve_source_chain() + logger.info(f"Hybrid search ({' → '.join(chain)}): {query}") + return await self.engine.search_with_fallback(query, chain, timeout, progress_callback) async def search_and_download_best(self, query: str, expected_track=None) -> Optional[str]: """ @@ -262,7 +319,7 @@ class DownloadOrchestrator: return None # 2. Filter and validate results - _streaming_sources = ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr') + _streaming_sources = ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') is_streaming = tracks[0].username in _streaming_sources if tracks else False if is_streaming and expected_track: @@ -312,7 +369,8 @@ class DownloadOrchestrator: elif is_streaming: filtered_results = tracks else: - filtered_results = self.soulseek.filter_results_by_quality_preference(tracks) if self.soulseek else tracks + soulseek = self.client('soulseek') + filtered_results = soulseek.filter_results_by_quality_preference(tracks) if soulseek else tracks if not filtered_results: logger.warning(f"No suitable quality results found for: {query}") @@ -335,102 +393,47 @@ class DownloadOrchestrator: Download a track using the appropriate client. Args: - username: Username (or "youtube" for YouTube) - filename: Filename or YouTube video ID + username: Source-name string for streaming sources + (e.g. ``'youtube'``, ``'tidal'``, ``'deezer_dl'``) + OR the actual slskd peer username for Soulseek. + filename: Filename / video ID / track ID encoding (source-specific) file_size: File size estimate Returns: download_id: Unique download ID for tracking """ - # Detect which client to use based on username - source_map = {'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz, - 'hifi': self.hifi, 'deezer_dl': self.deezer_dl, 'lidarr': self.lidarr} - source_names = {'youtube': 'YouTube', 'tidal': 'Tidal', 'qobuz': 'Qobuz', - 'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr'} - - if username in source_map: - client = source_map[username] + # Streaming sources are dispatched by name match; anything + # unrecognized falls through to Soulseek (peer username case). + spec = self.registry.get_spec(username) if username else None + if spec is not None and spec.name != 'soulseek': + client = self.registry.get(spec.name) if not client: - raise RuntimeError(f"{source_names[username]} download client not available (failed to initialize)") - logger.info(f"Downloading from {source_names[username]}: {filename}") + raise RuntimeError(f"{spec.display_name} download client not available (failed to initialize)") + logger.info(f"Downloading from {spec.display_name}: {filename}") return await client.download(username, filename, file_size) - else: - if not self.soulseek: - raise RuntimeError("Soulseek client not available (failed to initialize)") - logger.info(f"Downloading from Soulseek: {filename}") - return await self.soulseek.download(username, filename, file_size) + + soulseek = self.registry.get('soulseek') + if not soulseek: + raise RuntimeError("Soulseek client not available (failed to initialize)") + logger.info(f"Downloading from Soulseek: {filename}") + return await soulseek.download(username, filename, file_size) async def get_all_downloads(self) -> List[DownloadStatus]: - """ - Get all active downloads from all sources. - - Returns: - List of DownloadStatus objects - """ - # Get downloads from all available sources - all_downloads = [] - for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr]: - if client: - try: - all_downloads.extend(await client.get_all_downloads()) - except Exception: - pass - return all_downloads + """Aggregated view across every source. Delegates to the + engine, which iterates registered plugins.""" + return await self.engine.get_all_downloads() async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: - """ - Get status of a specific download. - - Args: - download_id: Download ID to query - - Returns: - DownloadStatus object or None if not found - """ - # Try each source until we find the download - for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr]: - if not client: - continue - try: - status = await client.get_download_status(download_id) - if status: - return status - except Exception: - pass - - return None + """Find a download by id across every source. Delegates to + the engine.""" + return await self.engine.get_download_status(download_id) async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool: - """ - Cancel an active download. - - Args: - download_id: Download ID to cancel - username: Username hint (optional) - remove: Whether to remove from active downloads - - Returns: - True if cancelled successfully - """ - # If username is provided, route directly to that source - source_map = {'youtube': self.youtube, 'tidal': self.tidal, 'qobuz': self.qobuz, - 'hifi': self.hifi, 'deezer_dl': self.deezer_dl, 'lidarr': self.lidarr} - if username in source_map: - client = source_map[username] - return await client.cancel_download(download_id, username, remove) if client else False - elif username: - return await self.soulseek.cancel_download(download_id, username, remove) if self.soulseek else False - - # Otherwise, try all available sources - for client in [self.soulseek, self.youtube, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr]: - if not client: - continue - try: - if await client.cancel_download(download_id, username, remove): - return True - except Exception: - pass - return False + """Cancel an active download. Delegates to the engine, which + handles source-hint routing (streaming source name → direct + plugin, unknown name → Soulseek as peer username, no hint → + try every plugin).""" + return await self.engine.cancel_download(download_id, username, remove) async def signal_download_completion(self, download_id: str, username: str, remove: bool = True) -> bool: """ @@ -445,39 +448,16 @@ class DownloadOrchestrator: True if successful """ # This is Soulseek-specific, so only call on Soulseek client - if not self.soulseek: + soulseek = self.client('soulseek') + if not soulseek: return False - return await self.soulseek.signal_download_completion(download_id, username, remove) + return await soulseek.signal_download_completion(download_id, username, remove) async def clear_all_completed_downloads(self) -> bool: - """ - Clear all completed downloads from both sources. - - Returns: - True if successful - """ - results = [] - for name, client in [ - ("soulseek", self.soulseek), - ("youtube", self.youtube), - ("tidal", self.tidal), - ("qobuz", self.qobuz), - ("hifi", self.hifi), - ("deezer_dl", self.deezer_dl), - ("lidarr", self.lidarr), - ]: - if not client: - continue - if hasattr(client, "is_configured") and not client.is_configured(): - logger.debug("Skipping %s clear_all_completed_downloads (not configured)", name) - continue - try: - results.append(await client.clear_all_completed_downloads()) - except Exception as exc: - logger.warning("%s clear_all_completed_downloads failed: %s", name, exc) - results.append(False) - - return all(results) if results else True + """Clear completed downloads from every source. Delegates + to the engine, which skips unconfigured plugins and treats + per-plugin failures as False (not an exception).""" + return await self.engine.clear_all_completed_downloads() # ===== Soulseek-specific methods (for backwards compatibility) ===== # These are internal methods that some parts of the codebase use directly @@ -495,9 +475,10 @@ class DownloadOrchestrator: Returns: API response """ - if not self.soulseek: + soulseek = self.client('soulseek') + if not soulseek: raise RuntimeError("Soulseek client not available (failed to initialize)") - return await self.soulseek._make_request(method, endpoint, **kwargs) + return await soulseek._make_request(method, endpoint, **kwargs) async def _make_direct_request(self, method: str, endpoint: str, **kwargs): """ @@ -512,9 +493,10 @@ class DownloadOrchestrator: Returns: API response """ - if not self.soulseek: + soulseek = self.client('soulseek') + if not soulseek: raise RuntimeError("Soulseek client not available (failed to initialize)") - return await self.soulseek._make_direct_request(method, endpoint, **kwargs) + return await soulseek._make_direct_request(method, endpoint, **kwargs) async def clear_all_searches(self) -> bool: """ @@ -523,7 +505,8 @@ class DownloadOrchestrator: Returns: True if successful """ - return await self.soulseek.clear_all_searches() if self.soulseek else True + soulseek = self.client('soulseek') + return await soulseek.clear_all_searches() if soulseek else True async def maintain_search_history_with_buffer(self, keep_searches: int = 50, trigger_threshold: int = 200) -> bool: """ @@ -536,15 +519,54 @@ class DownloadOrchestrator: Returns: True if successful """ - return await self.soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) if self.soulseek else True + soulseek = self.client('soulseek') + return await soulseek.maintain_search_history_with_buffer(keep_searches, trigger_threshold) if soulseek else True async def cancel_all_downloads(self) -> bool: - """Cancel and remove all downloads from all sources.""" + """Cancel and remove all downloads from all sources. + + Note: YouTube is intentionally excluded from this loop in the + legacy implementation — preserved here. (yt-dlp downloads + run as detached subprocesses and don't share the + ``cancel_all_downloads`` semantics the streaming sources + use.) Sources without ``cancel_all_downloads`` fall back to + ``clear_all_completed_downloads``. + """ ok = True - for client in [self.soulseek, self.tidal, self.qobuz, self.hifi, self.deezer_dl, self.lidarr]: - if client: - try: - await client.cancel_all_downloads() if hasattr(client, 'cancel_all_downloads') else await client.clear_all_completed_downloads() - except Exception: - ok = False + for name, client in self.registry.all_plugins(): + if name == 'youtube': + continue + try: + await client.cancel_all_downloads() if hasattr(client, 'cancel_all_downloads') else await client.clear_all_completed_downloads() + except Exception: + ok = False return ok + + +# --------------------------------------------------------------------------- +# Singleton accessor — mirrors Cin's metadata engine pattern +# (``get_metadata_engine()``). Callers that don't need a custom +# registry use this instead of instantiating DownloadOrchestrator +# directly. web_server.py constructs the singleton at startup and +# exposes it via the ``download_orchestrator`` global. +# --------------------------------------------------------------------------- + +_default_orchestrator: Optional['DownloadOrchestrator'] = None + + +def get_download_orchestrator() -> 'DownloadOrchestrator': + """Return (lazily creating) the process-wide DownloadOrchestrator + singleton. Mirrors the ``get_metadata_engine()`` pattern Cin used + for the metadata engine refactor.""" + global _default_orchestrator + if _default_orchestrator is None: + _default_orchestrator = DownloadOrchestrator() + return _default_orchestrator + + +def set_download_orchestrator(orchestrator: 'DownloadOrchestrator') -> None: + """Set the process-wide singleton. Used by web_server.py at boot + to install the orchestrator it constructs as the default for + callers that grab via ``get_download_orchestrator()``.""" + global _default_orchestrator + _default_orchestrator = orchestrator diff --git a/core/download_plugins/__init__.py b/core/download_plugins/__init__.py new file mode 100644 index 00000000..ed6a79b6 --- /dev/null +++ b/core/download_plugins/__init__.py @@ -0,0 +1,34 @@ +"""Download source plugin contract + registry. + +This package defines the canonical interface every download source +(Soulseek, YouTube, Tidal, Qobuz, HiFi, Deezer, Lidarr, SoundCloud, +and future additions like Usenet) must satisfy. The orchestrator +dispatches through this contract instead of hardcoded +`if self.youtube ... elif self.tidal ...` chains. + +This is the foundation step of a multi-commit refactor. Subsequent +commits extract shared logic (background download worker, search +query normalization, post-processing context building) into the +contract so adding a new source becomes a one-class plugin instead +of a 700+ LOC copy-paste loop. + +See `core/download_plugins/base.py` for the protocol contract and +`core/download_plugins/registry.py` for the dispatch entry point. +""" + +from core.download_plugins.base import DownloadSourcePlugin + +# NOTE: DownloadPluginRegistry is intentionally NOT re-exported here. +# Importing the registry triggers eager imports of every client class +# (see registry.py for why eager — test fixtures inject mock modules +# at collection time and we need real bindings before that). Clients +# inherit from DownloadSourcePlugin (Cin's review feedback — visible +# contract conformance), so importing the package via ``from +# core.download_plugins import DownloadSourcePlugin`` from a client +# file would create a circular import if registry came along for the +# ride. Callers that need the registry import it directly: +# from core.download_plugins.registry import DownloadPluginRegistry + +__all__ = [ + "DownloadSourcePlugin", +] diff --git a/core/download_plugins/base.py b/core/download_plugins/base.py new file mode 100644 index 00000000..d4604f40 --- /dev/null +++ b/core/download_plugins/base.py @@ -0,0 +1,134 @@ +"""Canonical contract every download source plugin must satisfy. + +`DownloadSourcePlugin` is a structural Protocol — any class that +implements these methods with matching signatures is automatically +treated as a download source. No inheritance required, no manual +registration required beyond the registry entry. + +The protocol is intentionally narrow — only the methods the +orchestrator dispatches generically across all sources. Source- +specific extras (Soulseek's slskd internals, Lidarr's album-only +flow, etc.) stay on the underlying client and are accessed through +the registry's typed accessor. + +This file is the FOUNDATION step. Existing client classes +(SoulseekClient, YouTubeClient, TidalDownloadClient, etc.) already +conform structurally — they grew the same shape independently +because every consumer site needed the same calls. This file just +makes the implicit contract explicit so: + +- Type checkers can flag drift if a new source forgets a method. +- The orchestrator can iterate plugins generically instead of + hardcoding `[self.soulseek, self.youtube, ...]` everywhere. +- Future PRs can move shared logic INTO the contract (a base + class with default implementations) without changing the + signature surface every consumer already depends on. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional, Protocol, Tuple, runtime_checkable + +# core.download_plugins.types owns the canonical TrackResult / +# AlbumResult / DownloadStatus dataclasses (lifted out of +# core.soulseek_client so plugins don't import from a sibling source +# just to satisfy the contract). We only need them for type +# annotations on this protocol; TYPE_CHECKING avoids a circular +# import once the clients themselves inherit from DownloadSourcePlugin +# (Cin's review feedback — clients explicitly declare conformance +# instead of relying on structural typing). +if TYPE_CHECKING: + from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult + + +@runtime_checkable +class DownloadSourcePlugin(Protocol): + """Structural contract for a download source. + + `runtime_checkable` lets `isinstance(client, DownloadSourcePlugin)` + work for the conformance test, but it ONLY checks method names — + not signatures or async-ness. The conformance test in + ``tests/test_download_plugin_conformance.py`` does the deeper + signature check. + """ + + # ------------------------------------------------------------------ + # Configuration / lifecycle + # ------------------------------------------------------------------ + + def is_configured(self) -> bool: + """Return True iff this source has the credentials / settings + it needs to function. Used by the orchestrator to skip + unconfigured sources during hybrid fallback.""" + ... + + async def check_connection(self) -> bool: + """Probe the source's API / endpoint. Return True if the + source is reachable. May make a live network call.""" + ... + + # ------------------------------------------------------------------ + # Search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback=None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + """Search the source for tracks (and albums where supported). + + Returns a tuple of (track_results, album_results). Either + list may be empty. Sources that don't expose album-level + search return ``[]`` as the second element. + """ + ... + + # ------------------------------------------------------------------ + # Download + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + """Kick off a download. Returns a download_id string the + caller can poll via ``get_download_status``. Returns ``None`` + if the source can't / won't handle this download. + + ``username`` is the source-name string for streaming sources + (e.g. ``'youtube'``, ``'tidal'``) and the actual slskd peer + username for Soulseek. ``filename`` is source-specific — + Soulseek file path, YouTube ``video_id||title``, Tidal / + Qobuz ``track_id||display``, etc. + """ + ... + + async def get_all_downloads(self) -> List[DownloadStatus]: + """Return live status of all downloads currently tracked by + this source. The orchestrator concatenates results from + every plugin to build the global download list.""" + ... + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + """Return status for a single download or ``None`` if this + source doesn't know about that download_id.""" + ... + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + """Cancel an active download. ``remove=True`` also drops + the row from the source's active-downloads tracking.""" + ... + + async def clear_all_completed_downloads(self) -> bool: + """Drop completed downloads from active tracking. Sources + that don't keep completed history return True with no-op.""" + ... diff --git a/core/download_plugins/registry.py b/core/download_plugins/registry.py new file mode 100644 index 00000000..ed62e899 --- /dev/null +++ b/core/download_plugins/registry.py @@ -0,0 +1,192 @@ +"""Plugin registry — single source of truth for which download +sources exist, what their canonical names are, and which client +class implements each. + +Replaces the orchestrator's hardcoded ``[self.soulseek, +self.youtube, self.tidal, ...]`` lists and ``source_map`` dicts +that historically had to be touched in 6+ places to add a source. +With the registry: + +- One ``register()`` call adds a source to every dispatch path. +- Iteration helpers replace hand-maintained lists. +- The orchestrator stays oblivious to source-specific quirks. +- Adding Usenet (planned) becomes a one-line registry entry plus + the new client class — no orchestrator changes. + +This is the foundation step. Subsequent commits move shared logic +(thread workers, search query normalization, post-processing +context building) out of the orchestrator and the per-source +clients into helpers the registry exposes. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Dict, Iterator, List, Optional, Tuple + +from utils.logging_config import get_logger + +from core.download_plugins.base import DownloadSourcePlugin + +# Eager client imports — keep the import-order behavior the orchestrator +# had before this refactor. Some integration tests inject mock modules +# into ``sys.modules`` at collection time (see +# ``tests/test_tidal_search_shortening.py``); making the registry's +# factories lazy-import would cause tidalapi-dependent code to bind to +# whichever ``tidalapi`` object happens to be in ``sys.modules`` at the +# moment ``DownloadOrchestrator()`` is constructed — which is later +# than the legacy module-top imports here. Importing everything at +# registry-load time pins the bindings the same way the legacy +# orchestrator did. +from core.deezer_download_client import DeezerDownloadClient +from core.hifi_client import HiFiClient +from core.lidarr_download_client import LidarrDownloadClient +from core.qobuz_client import QobuzClient +from core.soulseek_client import SoulseekClient +from core.soundcloud_client import SoundcloudClient +from core.tidal_download_client import TidalDownloadClient +from core.youtube_client import YouTubeClient + +logger = get_logger("download_plugins.registry") + + +@dataclass(frozen=True) +class PluginSpec: + """Static descriptor for a download source. The ``factory`` is + a zero-arg callable that builds the client instance — kept as a + callable rather than a class so each source can do its own + setup (e.g. SoulseekClient calls ``_setup_client`` after init, + Deezer reads ARL from config). ``aliases`` lets the registry + accept multiple historical names (e.g. ``deezer_dl`` is the + legacy alias for ``deezer``).""" + + name: str + factory: Callable[[], DownloadSourcePlugin] + display_name: str + aliases: Tuple[str, ...] = field(default_factory=tuple) + + +class DownloadPluginRegistry: + """Holds the live plugin instances + name → instance lookup. + + Construction is two-phase: + 1. Specs are registered (cheap — just stores callable refs). + 2. ``initialize()`` calls each factory once and stores the + resulting client. Failures are caught and logged so one + broken source doesn't take down the orchestrator (mirrors + the existing ``_safe_init`` behavior). + + Iteration helpers (``all_plugins``, ``configured_plugins``) + replace the hand-maintained lists scattered across the + orchestrator's ``get_all_downloads``, ``cancel_all_downloads``, + etc. so adding a source touches the registry alone. + """ + + def __init__(self) -> None: + self._specs: Dict[str, PluginSpec] = {} + self._instances: Dict[str, Optional[DownloadSourcePlugin]] = {} + self._init_failures: List[str] = [] + + def register(self, spec: PluginSpec) -> None: + """Register a plugin spec under its canonical name + each alias. + Aliases all resolve to the same instance after ``initialize``.""" + if spec.name in self._specs: + raise ValueError(f"Plugin already registered: {spec.name}") + self._specs[spec.name] = spec + + def initialize(self) -> None: + """Build every registered plugin's instance. Failures captured + in ``init_failures`` and the slot is set to None so the + orchestrator can skip unavailable sources without crashing.""" + for spec in self._specs.values(): + try: + instance = spec.factory() + self._instances[spec.name] = instance + except Exception as exc: + logger.error("%s download client failed to initialize: %s", spec.display_name, exc) + self._init_failures.append(spec.display_name) + self._instances[spec.name] = None + + @property + def init_failures(self) -> List[str]: + return list(self._init_failures) + + def get(self, name: str) -> Optional[DownloadSourcePlugin]: + """Resolve a name (or alias) to its plugin instance, or + None if the source failed to initialize / isn't registered.""" + if not name: + return None + # Direct hit + if name in self._instances: + return self._instances[name] + # Alias lookup + for spec in self._specs.values(): + if name in spec.aliases: + return self._instances.get(spec.name) + return None + + def get_spec(self, name: str) -> Optional[PluginSpec]: + if name in self._specs: + return self._specs[name] + for spec in self._specs.values(): + if name in spec.aliases: + return spec + return None + + def display_name(self, name: str) -> str: + spec = self.get_spec(name) + return spec.display_name if spec else name + + def names(self) -> List[str]: + """Canonical names of every registered source (regardless of + whether it initialized successfully).""" + return list(self._specs.keys()) + + def all_plugins(self) -> Iterator[Tuple[str, DownloadSourcePlugin]]: + """Yield (name, plugin) for every successfully-initialized + plugin. Replaces the orchestrator's hand-maintained client + lists in get_all_downloads / cancel_all_downloads / etc.""" + for name, instance in self._instances.items(): + if instance is not None: + yield name, instance + + def configured_plugins(self) -> Iterator[Tuple[str, DownloadSourcePlugin]]: + """Yield (name, plugin) for every initialized AND configured + plugin. Useful for hybrid mode and any operation that should + skip sources the user hasn't set up.""" + for name, instance in self.all_plugins(): + try: + if instance.is_configured(): + yield name, instance + except Exception: + continue + + +def build_default_registry() -> DownloadPluginRegistry: + """Construct the registry with SoulSync's eight built-in download + sources. Called once during orchestrator init. + + Adding a new source (Usenet, etc.) means adding one ``register`` + call here — no orchestrator changes required. + + The factory itself is just the class constructor — clients are + imported eagerly at the top of this module so they bind to the + real third-party libs (tidalapi, etc.) at import time, not at + factory-call time. See the import-block comment above for why. + """ + registry = DownloadPluginRegistry() + + registry.register(PluginSpec(name='soulseek', factory=SoulseekClient, display_name='Soulseek')) + registry.register(PluginSpec(name='youtube', factory=YouTubeClient, display_name='YouTube')) + registry.register(PluginSpec(name='tidal', factory=TidalDownloadClient, display_name='Tidal')) + registry.register(PluginSpec(name='qobuz', factory=QobuzClient, display_name='Qobuz')) + registry.register(PluginSpec(name='hifi', factory=HiFiClient, display_name='HiFi')) + # 'deezer_dl' is the legacy name used in config + per-source dispatch + # strings (e.g. orchestrator's ``source_map``). Canonical name is + # ``deezer`` so future-facing code reads naturally. + registry.register(PluginSpec(name='deezer', factory=DeezerDownloadClient, display_name='Deezer', + aliases=('deezer_dl',))) + registry.register(PluginSpec(name='lidarr', factory=LidarrDownloadClient, display_name='Lidarr')) + registry.register(PluginSpec(name='soundcloud',factory=SoundcloudClient, display_name='SoundCloud')) + + return registry diff --git a/core/download_plugins/types.py b/core/download_plugins/types.py new file mode 100644 index 00000000..46e250c3 --- /dev/null +++ b/core/download_plugins/types.py @@ -0,0 +1,199 @@ +"""Shared dataclasses for the download-plugin contract. + +Every download source returns the same shape for search hits and +download status — the four classes here are the canonical types +that the ``DownloadSourcePlugin`` Protocol exchanges. Living in +this neutral module (rather than ``core/soulseek_client.py`` where +they grew up by accident) means a new plugin doesn't have to import +from a sibling source just to satisfy the contract. + +Move history: extracted from ``core.soulseek_client`` so plugins +import from a neutral package per Cin's contract-first standard. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from core.imports.filename import parse_filename_metadata + + +@dataclass +class SearchResult: + """Base class for search results""" + username: str + filename: str + size: int + bitrate: Optional[int] + duration: Optional[int] # Duration in milliseconds (converted from slskd's seconds) + quality: str + free_upload_slots: int + upload_speed: int + queue_length: int + result_type: str = "track" # "track" or "album" + + @property + def quality_score(self) -> float: + quality_weights = { + 'flac': 1.0, + 'mp3': 0.8, + 'ogg': 0.7, + 'aac': 0.6, + 'wma': 0.5 + } + + base_score = quality_weights.get(self.quality.lower(), 0.3) + + if self.bitrate: + if self.bitrate >= 320: + base_score += 0.2 + elif self.bitrate >= 256: + base_score += 0.1 + elif self.bitrate < 128: + base_score -= 0.2 + + # Free upload slots + if self.free_upload_slots == 0: + base_score -= 0.15 + elif self.free_upload_slots > 0: + base_score += 0.05 + + # Upload speed in bytes/sec (tiered) + if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps + base_score += 0.15 + elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps + base_score += 0.10 + elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps + base_score += 0.05 + elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps + base_score -= 0.05 + + # Queue length (graduated penalty) + if self.queue_length > 50: + base_score -= 0.25 + elif self.queue_length > 20: + base_score -= 0.15 + elif self.queue_length > 10: + base_score -= 0.10 + + return min(base_score, 1.0) + + +@dataclass +class TrackResult(SearchResult): + """Individual track search result""" + artist: Optional[str] = None + title: Optional[str] = None + album: Optional[str] = None + track_number: Optional[int] = None + _source_metadata: Optional[Dict[str, Any]] = None + + def __post_init__(self): + self.result_type = "track" + # Try to extract metadata from filename if not provided + if not self.title or not self.artist: + self._parse_filename_metadata() + + def _parse_filename_metadata(self): + """Extract artist, title, album from filename patterns""" + parsed = parse_filename_metadata(self.filename) + if not self.artist and parsed.get("artist"): + self.artist = parsed["artist"] + if not self.title and parsed.get("title"): + self.title = parsed["title"] + if not self.album and parsed.get("album"): + self.album = parsed["album"] + if self.track_number is None: + track_number = parsed.get("track_number") + if track_number is not None: + self.track_number = track_number + + +@dataclass +class AlbumResult: + """Album/folder search result containing multiple tracks""" + username: str + album_path: str # Directory path + album_title: str + artist: Optional[str] + track_count: int + total_size: int + tracks: List[TrackResult] + dominant_quality: str # Most common quality in album + year: Optional[str] = None + free_upload_slots: int = 0 + upload_speed: int = 0 + queue_length: int = 0 + result_type: str = "album" + + @property + def quality_score(self) -> float: + """Calculate album quality score based on dominant quality and track count""" + quality_weights = { + 'flac': 1.0, + 'mp3': 0.8, + 'ogg': 0.7, + 'aac': 0.6, + 'wma': 0.5 + } + + base_score = quality_weights.get(self.dominant_quality.lower(), 0.3) + + # Bonus for complete albums (typically 8-15 tracks) + if 8 <= self.track_count <= 20: + base_score += 0.1 + elif self.track_count > 20: + base_score += 0.05 + + # Free upload slots + if self.free_upload_slots == 0: + base_score -= 0.15 + elif self.free_upload_slots > 0: + base_score += 0.05 + + # Upload speed in bytes/sec (tiered) + if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps + base_score += 0.15 + elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps + base_score += 0.10 + elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps + base_score += 0.05 + elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps + base_score -= 0.05 + + # Queue length (graduated penalty) + if self.queue_length > 50: + base_score -= 0.25 + elif self.queue_length > 20: + base_score -= 0.15 + elif self.queue_length > 10: + base_score -= 0.10 + + return min(base_score, 1.0) + + @property + def size_mb(self) -> int: + """Album size in MB""" + return self.total_size // (1024 * 1024) + + @property + def average_track_size_mb(self) -> float: + """Average track size in MB""" + if self.track_count > 0: + return self.size_mb / self.track_count + return 0 + + +@dataclass +class DownloadStatus: + id: str + filename: str + username: str + state: str + progress: float + size: int + transferred: int + speed: int + time_remaining: Optional[int] = None + file_path: Optional[str] = None diff --git a/core/downloads/cancel.py b/core/downloads/cancel.py index 6adecbc9..9536eda4 100644 --- a/core/downloads/cancel.py +++ b/core/downloads/cancel.py @@ -42,31 +42,31 @@ _TERMINAL_STATUSES = { } -def cancel_single_download(soulseek_client, run_async: Callable, +def cancel_single_download(download_orchestrator, run_async: Callable, download_id: str, username: str) -> bool: """Cancel one specific slskd download (with `remove=True`).""" - return run_async(soulseek_client.cancel_download(download_id, username, remove=True)) + return run_async(download_orchestrator.cancel_download(download_id, username, remove=True)) -def cancel_all_active(soulseek_client, run_async: Callable, +def cancel_all_active(download_orchestrator, run_async: Callable, sweep_callback: Callable[[], None]) -> tuple[bool, str]: """Cancel every active slskd download, clear the resulting ones, sweep dirs. Returns `(success, message)` so the route can map to the right HTTP shape. """ - cancel_success = run_async(soulseek_client.cancel_all_downloads()) + cancel_success = run_async(download_orchestrator.cancel_all_downloads()) if not cancel_success: return False, "Failed to cancel active downloads." - run_async(soulseek_client.clear_all_completed_downloads()) + run_async(download_orchestrator.clear_all_completed_downloads()) sweep_callback() return True, "All downloads cancelled and cleared." -def clear_finished_active(soulseek_client, run_async: Callable, +def clear_finished_active(download_orchestrator, run_async: Callable, sweep_callback: Callable[[], None]) -> bool: """Clear all terminal transfers from slskd, sweep dirs on success.""" - success = run_async(soulseek_client.clear_all_completed_downloads()) + success = run_async(download_orchestrator.clear_all_completed_downloads()) if success: sweep_callback() return success diff --git a/core/downloads/candidates.py b/core/downloads/candidates.py index 9a578e10..d74cfa9d 100644 --- a/core/downloads/candidates.py +++ b/core/downloads/candidates.py @@ -25,7 +25,7 @@ and notifies the lifecycle via `on_download_completed(success=False)` so the worker slot frees up. Lifted verbatim from web_server.py. Wide dependency surface -(soulseek_client, spotify_client, lifecycle callback, context-key helper, +(download_orchestrator, spotify_client, lifecycle callback, context-key helper, status updater, DB) all injected via `CandidatesDeps`. """ @@ -49,7 +49,7 @@ logger = logging.getLogger(__name__) @dataclass class CandidatesDeps: """Bundle of cross-cutting deps the candidate-fallback logic needs.""" - soulseek_client: Any + download_orchestrator: Any spotify_client: Any run_async: Callable[..., Any] get_database: Callable[[], Any] @@ -97,8 +97,8 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, if _bl_db.is_blacklisted(candidate.username, candidate.filename): logger.info(f"[Modal Worker] Skipping blacklisted source: {source_key}") continue - except Exception: - pass + except Exception as e: + logger.debug("blacklist check failed: %s", e) # CRITICAL: Add source to used_sources IMMEDIATELY to prevent race conditions # This must happen BEFORE starting download to prevent multiple retries from picking same source @@ -209,7 +209,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, # Initiate download logger.info(f"[Modal Worker] Starting download: {username} / {os.path.basename(filename)}") - download_id = deps.run_async(deps.soulseek_client.download(username, filename, size)) + download_id = deps.run_async(deps.download_orchestrator.download(username, filename, size)) if download_id: # Store context for post-processing with complete Spotify metadata (GUI PARITY) @@ -330,7 +330,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, logger.warning(f"[Modal Worker] Task {task_id} cancelled after download {download_id} started - attempting to cancel download") # Try to cancel the download immediately try: - deps.run_async(deps.soulseek_client.cancel_download(download_id, username, remove=True)) + deps.run_async(deps.download_orchestrator.cancel_download(download_id, username, remove=True)) logger.warning(f"Successfully cancelled active download {download_id}") except Exception as cancel_error: logger.error(f"Failed to cancel active download {download_id}: {cancel_error}") @@ -340,8 +340,8 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, deps.on_download_completed(batch_id, task_id, success=False) return False - # Store download information - use real download ID from soulseek_client - # CRITICAL FIX: Trust the download ID returned by soulseek_client.download() + # Store download information - use real download ID from download_orchestrator + # CRITICAL FIX: Trust the download ID returned by download_orchestrator.download() download_tasks[task_id]['download_id'] = download_id download_tasks[task_id]['username'] = username diff --git a/core/downloads/lifecycle.py b/core/downloads/lifecycle.py index aa784d8f..43e1e970 100644 --- a/core/downloads/lifecycle.py +++ b/core/downloads/lifecycle.py @@ -233,8 +233,8 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life 'title': track_info.get('track_name', ''), 'reason': track_info.get('failure_reason', 'Unknown'), }) - except Exception: - pass + except Exception as e: + logger.debug("download_failed emit failed: %s", e) # WISHLIST REMOVAL: Handle successful downloads for wishlist removal if success and task_id in download_tasks: @@ -364,8 +364,8 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life 'completed_tracks': str(successful_downloads), 'failed_tracks': str(failed_count), }) - except Exception: - pass + except Exception as e: + logger.debug("batch_complete emit failed: %s", e) # Update YouTube playlist phase to 'download_complete' if this is a YouTube playlist playlist_id = batch.get('playlist_id') @@ -569,8 +569,8 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo 'completed_tracks': str(successful_downloads), 'failed_tracks': str(failed_count), }) - except Exception: - pass + except Exception as e: + logger.debug("batch_complete emit failed: %s", e) else: logger.warning(f"[Completion Check V2] Batch {batch_id} already marked complete - skipping duplicate processing") return True # Already complete diff --git a/core/downloads/master.py b/core/downloads/master.py index e5c081d4..4620a467 100644 --- a/core/downloads/master.py +++ b/core/downloads/master.py @@ -42,7 +42,7 @@ logger = logging.getLogger(__name__) class MasterDeps: """Bundle of cross-cutting deps the master worker needs.""" config_manager: Any - soulseek_client: Any + download_orchestrator: Any run_async: Callable[..., Any] mb_worker: Any mb_release_cache: dict @@ -296,8 +296,8 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma }) if track_results: db_sh.update_sync_history_track_results(batch_id, json.dumps(track_results)) - except Exception: - pass + except Exception as e: + logger.debug("update sync_history track results failed: %s", e) is_auto_batch = False with tasks_lock: @@ -372,7 +372,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma _sr.info(f"[Album Pre-flight] Searching for '{artist_name} {album_name}'") logger.info(f"[Album Pre-flight] Searching Soulseek for complete album: '{artist_name} - {album_name}'") - slsk = deps.soulseek_client.soulseek if hasattr(deps.soulseek_client, 'soulseek') else deps.soulseek_client + slsk = deps.download_orchestrator.client('soulseek') if hasattr(deps.download_orchestrator, 'client') else deps.download_orchestrator # Try multiple query variations (banned keywords in artist/album name can return 0 results) album_queries = [f"{artist_name} {album_name}"] diff --git a/core/downloads/monitor.py b/core/downloads/monitor.py index c879257a..94b4a727 100644 --- a/core/downloads/monitor.py +++ b/core/downloads/monitor.py @@ -33,7 +33,7 @@ _run_post_processing_worker = None _start_next_batch_of_downloads = None _orphaned_download_keys = None missing_download_executor = None -soulseek_client = None +download_orchestrator = None def init( @@ -44,12 +44,12 @@ def init( start_next_batch_of_downloads, orphaned_download_keys, missing_download_executor_obj, - soulseek_client_obj, + download_orchestrator_obj, ): """Bind web_server-side helpers/globals so the class body can resolve them.""" global _make_context_key, _on_download_completed, _download_track_worker global _run_post_processing_worker, _start_next_batch_of_downloads - global _orphaned_download_keys, missing_download_executor, soulseek_client + global _orphaned_download_keys, missing_download_executor, download_orchestrator _make_context_key = make_context_key _on_download_completed = on_download_completed _download_track_worker = download_track_worker @@ -57,7 +57,7 @@ def init( _start_next_batch_of_downloads = start_next_batch_of_downloads _orphaned_download_keys = orphaned_download_keys missing_download_executor = missing_download_executor_obj - soulseek_client = soulseek_client_obj + download_orchestrator = download_orchestrator_obj class WebUIDownloadMonitor: @@ -199,7 +199,7 @@ class WebUIDownloadMonitor: if op[0] == 'cancel_download': _, download_id, username = op logger.debug(f"[Deferred] Cancelling download: {download_id} from {username}") - run_async(soulseek_client.cancel_download(download_id, username, remove=True)) + run_async(download_orchestrator.cancel_download(download_id, username, remove=True)) logger.debug(f"[Deferred] Successfully cancelled download {download_id}") elif op[0] == 'cleanup_orphan': _, context_key = op @@ -254,8 +254,9 @@ class WebUIDownloadMonitor: # Get Soulseek downloads from API transfers_data = None - if soulseek_active and soulseek_client and getattr(soulseek_client, 'soulseek', None) and soulseek_client.soulseek.base_url: - transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + _slsk = download_orchestrator.client('soulseek') if download_orchestrator and hasattr(download_orchestrator, 'client') else None + if soulseek_active and _slsk and _slsk.base_url: + transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads')) if transfers_data: for user_data in transfers_data: username = user_data.get('username', 'Unknown') @@ -266,30 +267,34 @@ class WebUIDownloadMonitor: key = _make_context_key(username, file_info.get('filename', '')) live_transfers[key] = file_info - # Also get non-Soulseek downloads (YouTube/Tidal/Qobuz/HiFi/Deezer/Lidarr) - # Call each client directly to avoid redundant slskd API call through orchestrator + # Also get non-Soulseek downloads via the engine — single + # cross-source aggregation, no per-source iteration. try: all_downloads = [] - for _dl_client in [soulseek_client.youtube, soulseek_client.tidal, soulseek_client.qobuz, - soulseek_client.hifi, soulseek_client.deezer_dl, soulseek_client.lidarr]: - if _dl_client: - try: - all_downloads.extend(run_async(_dl_client.get_all_downloads())) - except Exception: - pass + if download_orchestrator and hasattr(download_orchestrator, 'engine'): + try: + # Exclude soulseek — slskd transfers were already + # pulled via the transfers/downloads endpoint above. + # Without the exclude both fetch paths run, doubling + # the per-tick slskd API hit. + all_downloads = run_async( + download_orchestrator.engine.get_all_downloads(exclude=('soulseek',)) + ) + except Exception as e: + logger.debug("get_all_downloads failed: %s", e) for download in all_downloads: - key = _make_context_key(download.username, download.filename) - # Convert DownloadStatus to transfer dict format for monitor compatibility - live_transfers[key] = { - 'id': download.id, - 'filename': download.filename, - 'username': download.username, - 'state': download.state, - 'percentComplete': download.progress, - 'size': download.size, - 'bytesTransferred': download.transferred, - 'averageSpeed': download.speed, - } + key = _make_context_key(download.username, download.filename) + # Convert DownloadStatus to transfer dict format for monitor compatibility + live_transfers[key] = { + 'id': download.id, + 'filename': download.filename, + 'username': download.username, + 'state': download.state, + 'percentComplete': download.progress, + 'size': download.size, + 'bytesTransferred': download.transferred, + 'averageSpeed': download.speed, + } except Exception as yt_error: logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}") diff --git a/core/downloads/post_processing.py b/core/downloads/post_processing.py index 4a3df9d2..1b2d04d2 100644 --- a/core/downloads/post_processing.py +++ b/core/downloads/post_processing.py @@ -54,7 +54,7 @@ class PostProcessDeps: always live (no caching of pre-init Spotify clients etc). """ config_manager: Any - soulseek_client: Any + download_orchestrator: Any run_async: Callable docker_resolve_path: Callable[[str], str] extract_filename: Callable[[str], str] @@ -196,7 +196,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # Query the download orchestrator for the status which contains the real file path # CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id actual_download_id = task.get('download_id') or task_id - status = deps.run_async(deps.soulseek_client.get_download_status(actual_download_id)) + status = deps.run_async(deps.download_orchestrator.get_download_status(actual_download_id)) if status and status.file_path: real_path = status.file_path if os.path.exists(real_path): diff --git a/core/downloads/task_worker.py b/core/downloads/task_worker.py index d7708154..ebe5f716 100644 --- a/core/downloads/task_worker.py +++ b/core/downloads/task_worker.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) @dataclass class TaskWorkerDeps: """Bundle of cross-cutting deps the per-task download worker needs.""" - soulseek_client: Any + download_orchestrator: Any matching_engine: Any run_async: Callable try_source_reuse: Callable # (task_id, batch_id, track) -> bool @@ -210,7 +210,7 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke try: # Perform search with timeout - tracks_result, _ = deps.run_async(deps.soulseek_client.search(query, timeout=30)) + tracks_result, _ = deps.run_async(deps.download_orchestrator.search(query, timeout=30)) logger.debug(f"Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results") # CRITICAL: Check cancellation immediately after search returns @@ -260,7 +260,13 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke search_diagnostics.append(f'"{query}": {result_count} results, {len(candidates)} passed filters but download failed to start') else: search_diagnostics.append(f'"{query}": {result_count} results but none passed quality/artist filters') - all_raw_results.extend(tracks_result[:20]) # Keep top results for review + # Strip SoundCloud preview snippets before caching for the + # review modal — the user can't pick something useful from + # a 30s preview clip, and clicking one bypasses validation + # and downloads it anyway. + from core.downloads.validation import filter_soundcloud_previews + _filtered_raw = filter_soundcloud_previews(tracks_result[:20], track) + all_raw_results.extend(_filtered_raw) else: search_diagnostics.append(f'"{query}": no results found') @@ -272,22 +278,23 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke # === HYBRID FALLBACK: If primary source failed, try remaining sources directly === # The orchestrator's hybrid search stops at the first source with results, even if # those results all fail quality filtering. Try remaining sources individually. - if getattr(deps.soulseek_client, 'mode', '') == 'hybrid': + if getattr(deps.download_orchestrator, 'mode', '') == 'hybrid': try: - orch = deps.soulseek_client + orch = deps.download_orchestrator hybrid_order = getattr(orch, 'hybrid_order', None) or [] if not hybrid_order: primary = getattr(orch, 'hybrid_primary', 'soulseek') secondary = getattr(orch, 'hybrid_secondary', '') hybrid_order = [primary, secondary] if secondary and secondary != primary else [primary] + # Resolve via the orchestrator's generic accessor — the + # legacy per-source attrs were dropped in the registry + # refactor, so getattr(orch, 'soulseek', None) etc. all + # silently returned None and the fallback never fired. source_clients = { - 'soulseek': getattr(orch, 'soulseek', None), - 'youtube': getattr(orch, 'youtube', None), - 'tidal': getattr(orch, 'tidal', None), - 'qobuz': getattr(orch, 'qobuz', None), - 'hifi': getattr(orch, 'hifi', None), - 'deezer_dl': getattr(orch, 'deezer_dl', None), + name: orch.client(name) + for name in ('soulseek', 'youtube', 'tidal', 'qobuz', + 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') } # The orchestrator tried sources in order but stopped at the first with results. diff --git a/core/downloads/validation.py b/core/downloads/validation.py index fff78401..17171535 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -1,7 +1,7 @@ """Soulseek/streaming candidate validation — lifted from web_server.py. Body is byte-identical to the original. ``matching_engine`` and -``soulseek_client`` are injected via init() because both are +``download_orchestrator`` are injected via init() because both are constructed in web_server.py and referenced by name throughout the body. """ @@ -14,14 +14,51 @@ logger = logging.getLogger(__name__) # Injected at runtime via init(). matching_engine = None -soulseek_client = None +download_orchestrator = None -def init(matching_engine_obj, soulseek_client_obj): +def init(matching_engine_obj, download_orchestrator_obj): """Bind the matching engine and download orchestrator from web_server.""" - global matching_engine, soulseek_client + global matching_engine, download_orchestrator matching_engine = matching_engine_obj - soulseek_client = soulseek_client_obj + download_orchestrator = download_orchestrator_obj + + +def filter_soundcloud_previews(results, expected_track): + """Drop SoundCloud preview snippets so they never reach the cache, + the modal, or the auto-download attempt. + + SoundCloud serves a ~30s preview clip for tracks gated behind Go+ / + login. yt-dlp accepts the preview as the download payload, the + integrity check catches the truncated file, but the user just sees + "all candidates failed" with previews still listed in the modal + (and clickable for manual retry, which downloads another preview). + + Filter at every spot raw search results enter the task: validation + scoring, modal-cache fallback when validation drops everything, + and the not-found raw-results cache. Keep candidates that genuinely + are short (intros, sound effects) when the expected track is also + short. + """ + if not results or not expected_track: + return results + expected_ms = getattr(expected_track, 'duration_ms', 0) or 0 + if expected_ms <= 0: + return results + expected_secs = expected_ms / 1000.0 + if expected_secs <= 60: + return results + + def _is_preview(r): + if getattr(r, 'username', None) != 'soundcloud': + return False + cand_ms = getattr(r, 'duration', None) or 0 + if cand_ms <= 0: + return False + cand_secs = cand_ms / 1000.0 + return cand_secs < 35 or cand_secs < expected_secs * 0.5 + + return [r for r in results if not _is_preview(r)] def get_valid_candidates(results, spotify_track, query): @@ -33,9 +70,16 @@ def get_valid_candidates(results, spotify_track, query): if not results: return [] - # Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer) return structured API results + # Pre-filter: drop SoundCloud preview snippets when expected + # duration is non-trivially long. Same helper is also applied at + # the modal-cache fallback path so previews never reach the UI. + results = filter_soundcloud_previews(results, spotify_track) + if not results: + return [] + + # Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer, SoundCloud) return structured API results # with proper artist/title metadata — score using the same matching engine as Soulseek - _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl") + _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud") if results[0].username in _streaming_sources: source_label = results[0].username.replace('_dl', '').title() expected_artists = spotify_track.artists if spotify_track else [] @@ -45,7 +89,12 @@ def get_valid_candidates(results, spotify_track, query): # Detect if the expected track is a specific version (live, remix, acoustic, etc.) expected_title_lower = (expected_title or '').lower() _version_keywords = ['remix', 'live', 'acoustic', 'instrumental', 'radio edit', - 'extended', 'slowed', 'sped up', 'reverb', 'karaoke'] + 'extended', 'slowed', 'sped up', 'reverb', 'karaoke', + # Producer-tag noise common on SoundCloud — "type + # beat" is an instrumental track produced in + # someone's style, tagged with the artist name to + # game search. NEVER the real song. + 'type beat'] expected_is_version = any(kw in expected_title_lower for kw in _version_keywords) scored = [] @@ -151,8 +200,8 @@ def get_valid_candidates(results, spotify_track, query): quality_filtered_candidates = initial_candidates else: # Filter by user's quality profile before artist verification (Soulseek only) - # Use existing soulseek_client to avoid re-initializing (which accesses download_path filesystem) - quality_filtered_candidates = soulseek_client.soulseek.filter_results_by_quality_preference(initial_candidates) + # Use existing download_orchestrator to avoid re-initializing (which accesses download_path filesystem) + quality_filtered_candidates = download_orchestrator.client('soulseek').filter_results_by_quality_preference(initial_candidates) # IMPORTANT: Respect empty results from quality filter # If user has strict quality requirements (e.g., FLAC-only with fallback disabled), diff --git a/core/downloads/wishlist_failed.py b/core/downloads/wishlist_failed.py index f3e7d936..1b1f30bb 100644 --- a/core/downloads/wishlist_failed.py +++ b/core/downloads/wishlist_failed.py @@ -2,7 +2,7 @@ Body is byte-identical to the original. Wishlist helpers are direct imports from core.wishlist.*; runtime state comes from -core.runtime_state; automation_engine, soulseek_client, and the +core.runtime_state; automation_engine, download_orchestrator, and the sweep helper are injected via init() because they are constructed in web_server.py. """ @@ -29,15 +29,15 @@ logger = logging.getLogger(__name__) # Injected at runtime via init(). automation_engine = None -soulseek_client = None +download_orchestrator = None _sweep_empty_download_directories = None -def init(engine, soulseek_client_obj, sweep_fn): +def init(engine, download_orchestrator_obj, sweep_fn): """Bind shared singletons + the sweep helper from web_server.""" - global automation_engine, soulseek_client, _sweep_empty_download_directories + global automation_engine, download_orchestrator, _sweep_empty_download_directories automation_engine = engine - soulseek_client = soulseek_client_obj + download_orchestrator = download_orchestrator_obj _sweep_empty_download_directories = sweep_fn @@ -146,8 +146,8 @@ def _process_failed_tracks_to_wishlist_exact(batch_id): 'title': track_name, 'reason': failed_track_info.get('failure_reason', ''), }) - except Exception: - pass + except Exception as e: + logger.debug("emit wishlist_item_added failed: %s", e) else: logger.error(f"[Wishlist Processing] Failed to add {track_name} to wishlist") @@ -185,7 +185,7 @@ def _process_failed_tracks_to_wishlist_exact(batch_id): # Auto-cleanup: Clear completed downloads from slskd try: logger.info(f"[Auto-Cleanup] Clearing completed downloads from slskd after batch {batch_id}") - run_async(soulseek_client.clear_all_completed_downloads()) + run_async(download_orchestrator.clear_all_completed_downloads()) logger.info("[Auto-Cleanup] Completed downloads cleared from slskd") except Exception as cleanup_error: logger.warning(f"[Auto-Cleanup] Failed to clear completed downloads: {cleanup_error}") diff --git a/core/enrichment/__init__.py b/core/enrichment/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/enrichment/api.py b/core/enrichment/api.py new file mode 100644 index 00000000..e4d097d7 --- /dev/null +++ b/core/enrichment/api.py @@ -0,0 +1,156 @@ +"""Generic Flask routes for enrichment-bubble status / pause / resume. + +Replaces 30 near-identical per-service routes that web_server.py used +to hand-roll. The blueprint reads the registry in ``core.enrichment.services`` +and dispatches: + + GET /api/enrichment//status + POST /api/enrichment//pause + POST /api/enrichment//resume + +A 404 is returned for unknown service ids. Per-service quirks (Spotify +rate-limit guard, auto-pause token cleanup, persisted-pause config keys) +are encoded as data on the ``EnrichmentService`` descriptor — there is +no branching on service id inside this module. +""" + +from __future__ import annotations + +from typing import Any, Callable, Optional + +from flask import Blueprint, jsonify + +from core.enrichment.services import EnrichmentService, get_service +from utils.logging_config import get_logger + + +logger = get_logger("enrichment.api") + + +# Hooks the host wires up so the blueprint can persist pause state and +# clean up auto-pause / yield-override sets without circular imports. +_config_set: Optional[Callable[[str, Any], None]] = None +_auto_paused_discard: Optional[Callable[[str], None]] = None +_yield_override_add: Optional[Callable[[str], None]] = None + + +def configure( + *, + config_set: Optional[Callable[[str, Any], None]] = None, + auto_paused_discard: Optional[Callable[[str], None]] = None, + yield_override_add: Optional[Callable[[str], None]] = None, +) -> None: + """Wire host-side mutators that the generic routes call after pause/resume. + + Each is optional — pass None for hosts that don't have a corresponding + mechanism (e.g. tests). + """ + global _config_set, _auto_paused_discard, _yield_override_add + _config_set = config_set + _auto_paused_discard = auto_paused_discard + _yield_override_add = yield_override_add + + +def _persist_paused(service: EnrichmentService, paused: bool) -> None: + if not service.config_paused_key or _config_set is None: + return + try: + _config_set(service.config_paused_key, paused) + except Exception as e: + logger.warning( + "Persisting pause flag for %s failed: %s", service.id, e + ) + + +def _drop_auto_pause_marker(service: EnrichmentService) -> None: + if service.auto_pause_token is None or _auto_paused_discard is None: + return + try: + _auto_paused_discard(service.auto_pause_token) + except Exception as e: + logger.debug("auto-pause marker discard: %s", e) + + +def _add_yield_override(service: EnrichmentService) -> None: + if service.auto_pause_token is None or _yield_override_add is None: + return + try: + _yield_override_add(service.auto_pause_token) + except Exception as e: + logger.debug("yield override add: %s", e) + + +def create_blueprint() -> Blueprint: + """Build the Flask blueprint — call once during host startup.""" + bp = Blueprint('enrichment_api', __name__) + + @bp.route('/api/enrichment//status', methods=['GET']) + def enrichment_status(service_id: str): + service = get_service(service_id) + if service is None: + return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404 + try: + worker = service.get_worker() + if worker is None: + return jsonify(service.fallback_status()), 200 + return jsonify(worker.get_stats()), 200 + except Exception as e: + logger.error("Error getting %s enrichment status: %s", service.id, e) + return jsonify({'error': str(e)}), 500 + + @bp.route('/api/enrichment//pause', methods=['POST']) + def enrichment_pause(service_id: str): + service = get_service(service_id) + if service is None: + return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404 + worker = service.get_worker() + if worker is None: + return jsonify({ + 'error': f'{service.display_name} enrichment worker not initialized', + }), 400 + try: + worker.pause() + _persist_paused(service, True) + _drop_auto_pause_marker(service) + logger.info("%s worker paused via UI", service.display_name) + return jsonify({'status': 'paused'}), 200 + except Exception as e: + logger.error("Error pausing %s worker: %s", service.id, e) + return jsonify({'error': str(e)}), 500 + + @bp.route('/api/enrichment//resume', methods=['POST']) + def enrichment_resume(service_id: str): + service = get_service(service_id) + if service is None: + return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404 + worker = service.get_worker() + if worker is None: + return jsonify({ + 'error': f'{service.display_name} enrichment worker not initialized', + }), 400 + # Pre-resume guard (e.g. Spotify rate-limit ban). Returns + # (http_status, error_message) when blocking, None when ok. + if service.pre_resume_check is not None: + try: + blocked = service.pre_resume_check() + except Exception as e: + logger.error("Pre-resume check for %s raised: %s", service.id, e) + blocked = None + if blocked is not None: + http_status, message = blocked + payload: dict = {'error': message} + if http_status == 429: + payload['rate_limited'] = True + return jsonify(payload), http_status + try: + worker.resume() + _persist_paused(service, False) + _drop_auto_pause_marker(service) + _add_yield_override(service) + logger.info("%s worker resumed via UI", service.display_name) + return jsonify({'status': 'running'}), 200 + except Exception as e: + logger.error("Error resuming %s worker: %s", service.id, e) + return jsonify({'error': str(e)}), 500 + + return bp diff --git a/core/enrichment/manual_match_honoring.py b/core/enrichment/manual_match_honoring.py new file mode 100644 index 00000000..734d2cd4 --- /dev/null +++ b/core/enrichment/manual_match_honoring.py @@ -0,0 +1,128 @@ +"""Honor manually-matched source IDs in per-source enrichment workers. + +GitHub issue #501 (@Tacobell444): every per-source enrichment worker's +``_process_*_individual`` method ran a fuzzy text search on the album / +track name and overwrote the stored source ID with whatever the search +returned. If the user had manually matched an album to a specific source +ID (e.g. set ``albums.spotify_album_id = 'ABC'`` via the match-chip UI), +the next "Enrich" click would search by name → pick a different result +→ overwrite the manual match with the wrong ID, OR fail to match +anything and revert the status to ``not_found``. + +This module lifts the "honor stored ID" fast path into one shared +helper. Each per-source worker (Spotify / iTunes / Deezer / Discogs / +MusicBrainz / AudioDB / Tidal / Qobuz) calls it before falling back +to its existing search-by-name flow. Same fix in 8 workers gets +exactly one implementation; per-worker variability (column name, +client fetch method, response shape) plugs in via callbacks. + +Lift what's truly shared. Caller knows its own column + client +method + update logic; the helper just orchestrates. +""" + +from __future__ import annotations + +from typing import Any, Callable, Optional + +from utils.logging_config import get_logger + +logger = get_logger("enrichment.manual_match_honoring") + + +def _read_id_column(db, entity_table: str, entity_id, id_column: str) -> Optional[str]: + """Read the stored source ID for one entity. Returns None when the + column is empty / unset.""" + if entity_table not in ('albums', 'tracks', 'artists'): + # Defensive: we only operate on these three. Avoids SQL injection + # via a bad table name (id_column is also restricted to known + # column names by callers but defense in depth never hurts). + return None + conn = db._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + f"SELECT {id_column} FROM {entity_table} WHERE id = ?", + (entity_id,), + ) + row = cursor.fetchone() + finally: + conn.close() + if not row: + return None + value = row[0] if not hasattr(row, 'keys') else row[id_column] + return str(value).strip() if value else None + + +def honor_stored_match( + *, + db, + entity_table: str, + entity_id, + id_column: str, + client_fetch_fn: Callable[[str], Any], + on_match_fn: Callable[[Any, str, Any], None], + log_prefix: str = '', +) -> bool: + """Fast-path enrichment via a stored source ID — preserves manual + matches. + + Args: + db: ``MusicDatabase`` instance (for the column read). + entity_table: ``'albums'``, ``'tracks'``, or ``'artists'``. + entity_id: Library DB ID of the entity to enrich. + id_column: Column on ``entity_table`` that stores the source- + specific ID (``spotify_album_id`` / ``itunes_album_id`` / + ``deezer_id`` / etc). + client_fetch_fn: Callable taking the stored ID and returning + the source's raw response (Album dataclass, dict, or + whatever the client returns). Typically + ``self.client.get_album`` or ``self.client.get_track``. + on_match_fn: Worker callback invoked with + ``(entity_id, stored_id, api_response)`` to apply the + metadata refresh. Worker knows the response shape; helper + doesn't. + log_prefix: Display name for log lines (``'Spotify'`` / + ``'iTunes'`` / etc). + + Returns: + True if a stored ID was found AND the fetch returned data AND + the on-match callback ran. Caller skips its search-by-name + flow and counts a match. + + False if no stored ID is set, the fetch failed, or the fetch + returned empty. Caller falls through to its existing search- + by-name flow (the legacy behavior for un-matched entities). + + Notes: + - Exceptions in ``client_fetch_fn`` are caught and logged at + warning level — caller falls through to search. + - Exceptions in ``on_match_fn`` propagate (those are real + DB errors the worker should know about). + """ + stored_id = _read_id_column(db, entity_table, entity_id, id_column) + if not stored_id: + return False + + try: + api_data = client_fetch_fn(stored_id) + except Exception as exc: + logger.warning( + f"[{log_prefix}] Stored-ID fetch failed for " + f"{entity_table[:-1]} #{entity_id} (id={stored_id}): {exc}" + ) + return False + + if not api_data: + logger.debug( + f"[{log_prefix}] Stored ID {stored_id} for " + f"{entity_table[:-1]} #{entity_id} returned empty data — " + f"falling through to search-by-name" + ) + return False + + on_match_fn(entity_id, stored_id, api_data) + logger.info( + f"[{log_prefix}] Honored manual match: " + f"{entity_table[:-1]} #{entity_id} → {id_column}={stored_id}" + ) + return True diff --git a/core/enrichment/services.py b/core/enrichment/services.py new file mode 100644 index 00000000..c32102a8 --- /dev/null +++ b/core/enrichment/services.py @@ -0,0 +1,125 @@ +"""Registry of enrichment workers exposed via the dashboard bubble UI. + +Every "bubble" on the dashboard (MusicBrainz, Spotify, iTunes, Last.fm, +Genius, Deezer, Discogs, AudioDB, Tidal, Qobuz) used to have its own +copy-pasted ``status`` / ``pause`` / ``resume`` Flask routes — 30 routes +that differed only in the worker reference and a couple of per-service +quirks. This module collapses them into a single ``EnrichmentService`` +descriptor + registry so the generic blueprint in ``core.enrichment.api`` +can drive every bubble from one place. + +Hydrabase (P2P mirror) and SoulID (entity ID generation) are intentionally +out of scope here — their workers report fundamentally different status +shapes and don't share the bubble pause/resume contract. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Tuple + + +# Default status payload shape returned when a worker isn't initialized. +# Mirrors the shape every per-service route used to inline before this +# refactor; UI consumers depend on these exact keys. +_DEFAULT_STATUS_FALLBACK: Dict[str, Any] = { + 'enabled': False, + 'running': False, + 'paused': False, + 'current_item': None, + 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, + 'progress': {}, +} + + +@dataclass +class EnrichmentService: + """Descriptor for one enrichment worker exposed via the dashboard. + + The dashboard talks to every worker through three identical-looking + endpoints (status / pause / resume). The variation between services + is captured here as data, not branching code: + + - ``worker_getter`` returns the live worker reference (or None when + initialization failed). Lazy so the registry can be defined before + web_server.py finishes module-level imports. + - ``config_paused_key`` is the ``config_manager`` key that persists + the user's pause / resume choice across restarts. Empty string + means "do not persist" (Hydrabase historically did this). + - ``pre_resume_check`` runs before resume — return ``(http_status, + error_message)`` to short-circuit (Spotify uses this for the + rate-limit guard). + - ``auto_pause_token`` matches an entry in + ``_download_auto_paused`` / ``_download_yield_override`` so the + pause/resume routes can clean those up correctly. None means + this service doesn't participate in the auto-pause-during-download + mechanism. + - ``extra_status_defaults`` is merged into the fallback status + payload (Tidal / Qobuz add ``'authenticated': False``). + """ + + id: str + display_name: str + worker_getter: Callable[[], Any] + config_paused_key: str = '' + pre_resume_check: Optional[Callable[[], Optional[Tuple[int, str]]]] = None + auto_pause_token: Optional[str] = None + extra_status_defaults: Dict[str, Any] = field(default_factory=dict) + + def get_worker(self) -> Any: + """Resolve the worker reference (None if init failed).""" + try: + return self.worker_getter() + except Exception: + return None + + def fallback_status(self) -> Dict[str, Any]: + """Return the shape we serve when the worker isn't initialized.""" + payload = dict(_DEFAULT_STATUS_FALLBACK) + # stats dict is shared — copy so callers can't mutate the module + # default. + payload['stats'] = dict(_DEFAULT_STATUS_FALLBACK['stats']) + if self.extra_status_defaults: + payload.update(self.extra_status_defaults) + return payload + + +# Module-level registry. Populated by ``register_services`` so the host +# (web_server.py) can wire its module-local worker globals + downstream +# state collections (auto-pause sets, rate-limit guard) without circular +# imports. +_REGISTRY: Dict[str, EnrichmentService] = {} + + +def register_services(services: List[EnrichmentService]) -> None: + """Replace the active service registry. + + The host registers all services in one call after its workers are + initialized. Re-registering is allowed (used by tests) — clears the + previous set. + """ + _REGISTRY.clear() + for svc in services: + if not svc.id: + raise ValueError("EnrichmentService.id must be non-empty") + _REGISTRY[svc.id] = svc + + +def get_service(service_id: str) -> Optional[EnrichmentService]: + """Return the registered service with this id, or None.""" + return _REGISTRY.get(service_id) + + +def all_services() -> List[EnrichmentService]: + """Return every registered service in registration order.""" + return list(_REGISTRY.values()) + + +def all_service_ids() -> List[str]: + """Return the ids of every registered service.""" + return list(_REGISTRY.keys()) + + +def clear_registry() -> None: + """Wipe the registry. Test-only — production code uses ``register_services``.""" + _REGISTRY.clear() diff --git a/core/genius_worker.py b/core/genius_worker.py index 63de47a6..e2e731fd 100644 --- a/core/genius_worker.py +++ b/core/genius_worker.py @@ -154,8 +154,8 @@ class GeniusWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null-id item type lookup: %s", e) continue self._process_item(item) @@ -367,8 +367,8 @@ class GeniusWorker: if song_url: try: lyrics = self.client.get_lyrics(song_url) - except Exception: - pass + except Exception as _e: + logger.debug("genius lyrics scrape: %s", _e) self._update_track(track_id, full_song, full_song, lyrics) self.stats['matched'] += 1 logger.info(f"Enriched track '{track_name}' from existing Genius ID: {existing_id}") diff --git a/core/hifi_client.py b/core/hifi_client.py index 7b3a8823..8fa51cdd 100644 --- a/core/hifi_client.py +++ b/core/hifi_client.py @@ -33,7 +33,7 @@ import requests as http_requests from utils.logging_config import get_logger from config.settings import config_manager -from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus +from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus logger = get_logger("hifi_client") @@ -86,7 +86,10 @@ DEFAULT_INSTANCES = [ ] -class HiFiClient: +from core.download_plugins.base import DownloadSourcePlugin + + +class HiFiClient(DownloadSourcePlugin): """ HiFi API client for searching and downloading lossless music. Uses public hifi-api instances (Tidal backend) — no auth required. @@ -110,9 +113,7 @@ class HiFiClient: 'Accept': 'application/json', }) - self.active_downloads: Dict[str, Dict[str, Any]] = {} - self._download_lock = threading.Lock() - + self._engine = None self.shutdown_check = None self._last_api_call = 0 @@ -125,6 +126,9 @@ class HiFiClient: def set_shutdown_check(self, check_callable): self.shutdown_check = check_callable + def set_engine(self, engine): + self._engine = engine + def _load_instances_from_db(self): try: from database.music_database import get_database @@ -282,24 +286,30 @@ class HiFiClient: def _parse_track(self, item: dict) -> Dict: artist_name = 'Unknown Artist' + artist_id = None artists_raw = item.get('artists', item.get('artist')) if isinstance(artists_raw, list): names = [] for a in artists_raw: if isinstance(a, dict): names.append(a.get('name', '')) + if artist_id is None: + artist_id = a.get('id') elif isinstance(a, str): names.append(a) artist_name = ', '.join(n for n in names if n) or 'Unknown Artist' elif isinstance(artists_raw, dict): artist_name = artists_raw.get('name', 'Unknown Artist') + artist_id = artists_raw.get('id') elif isinstance(artists_raw, str): artist_name = artists_raw album_raw = item.get('album', {}) album_name = '' + album_id = None if isinstance(album_raw, dict): album_name = album_raw.get('title', album_raw.get('name', '')) + album_id = album_raw.get('id') elif isinstance(album_raw, str): album_name = album_raw @@ -308,12 +318,16 @@ class HiFiClient: return { 'id': item.get('id'), + 'artist_id': artist_id, + 'album_id': album_id, 'title': item.get('title', item.get('name', 'Unknown')), 'artist': artist_name, 'album': album_name, 'duration_ms': int(duration_ms) if duration_ms else 0, 'track_number': item.get('trackNumber', item.get('track_number')), 'isrc': item.get('isrc'), + 'bpm': item.get('bpm'), + 'copyright': item.get('copyright'), 'explicit': item.get('explicit', False), 'quality': item.get('audioQuality', item.get('quality', '')), } @@ -549,74 +563,46 @@ class HiFiClient: title=track.get('title'), album=track.get('album'), track_number=track.get('track_number'), + _source_metadata={ + 'source': 'hifi', + 'track_id': track.get('id'), + 'artist_id': track.get('artist_id'), + 'album_id': track.get('album_id'), + 'isrc': track.get('isrc'), + 'bpm': track.get('bpm'), + 'copyright': track.get('copyright'), + }, ) async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]: + if '||' not in filename: + logger.error(f"Invalid filename format: {filename}") + return None + if self._engine is None: + # Raise rather than return None so the orchestrator's + # download_with_fallback surfaces a real warning + tries + # the next source. Returning None silently dropped the + # download with no user feedback (per JohnBaumb). + raise RuntimeError("HiFi client has no engine reference — cannot dispatch download") + + track_id_str, display_name = filename.split('||', 1) try: - if '||' not in filename: - logger.error(f"Invalid filename format: {filename}") - return None - - track_id_str, display_name = filename.split('||', 1) - try: - track_id = int(track_id_str) - except ValueError: - logger.error(f"Invalid track ID: {track_id_str}") - return None - - download_id = str(uuid.uuid4()) - - with self._download_lock: - self.active_downloads[download_id] = { - 'id': download_id, - 'filename': filename, - 'username': 'hifi', - 'state': 'Initializing', - 'progress': 0.0, - 'size': 0, - 'transferred': 0, - 'speed': 0, - 'time_remaining': None, - 'track_id': track_id, - 'display_name': display_name, - 'file_path': None, - } - - thread = threading.Thread( - target=self._download_worker, - args=(download_id, track_id, display_name), - daemon=True, - ) - thread.start() - - return download_id - - except Exception as e: - logger.error(f"Failed to start HiFi download: {e}") + track_id = int(track_id_str) + except ValueError: + logger.error(f"Invalid track ID: {track_id_str}") return None - def _download_worker(self, download_id: str, track_id: int, display_name: str): - try: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'InProgress, Downloading' - - file_path = self._download_sync(download_id, track_id, display_name) - - with self._download_lock: - if download_id in self.active_downloads: - if file_path: - self.active_downloads[download_id]['state'] = 'Completed, Succeeded' - self.active_downloads[download_id]['progress'] = 100.0 - self.active_downloads[download_id]['file_path'] = file_path - else: - self.active_downloads[download_id]['state'] = 'Errored' - - except Exception as e: - logger.error(f"HiFi download worker failed for {download_id}: {e}") - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' + return self._engine.worker.dispatch( + source_name='hifi', + target_id=track_id, + display_name=display_name, + original_filename=filename, + impl_callable=self._download_sync, + extra_record_fields={ + 'track_id': track_id, + 'display_name': display_name, + }, + ) def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]: quality_key = config_manager.get('hifi_download.quality', 'lossless') @@ -657,9 +643,8 @@ class HiFiClient: speed_start = time.time() segments_completed = 0 - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['size'] = 0 + if self._engine is not None: + self._engine.update_record('hifi', download_id, {'size': 0}) with intermediate_path.open('wb') as output_file: if init_uri: @@ -758,80 +743,77 @@ class HiFiClient: def _update_download_progress(self, download_id: str, downloaded: int, segments_completed: int, total_segments: int, speed_start: float): - with self._download_lock: - if download_id not in self.active_downloads: - return - info = self.active_downloads[download_id] - info['transferred'] = downloaded + if self._engine is None: + return + record = self._engine.get_record('hifi', download_id) + if record is None: + return - now = time.time() - elapsed_total = now - speed_start - speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0 - info['speed'] = speed + now = time.time() + elapsed_total = now - speed_start + speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0 - if total_segments > 0: - progress = (segments_completed / total_segments) * 100 - info['progress'] = round(min(progress, 99.9), 1) + progress = record.get('progress', 0.0) + if total_segments > 0: + progress = round(min((segments_completed / total_segments) * 100, 99.9), 1) - time_remaining = None - if speed > 0: - remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded - if remaining_bytes > 0: - time_remaining = int(remaining_bytes / speed) - info['time_remaining'] = time_remaining + time_remaining = None + if speed > 0: + remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded + if remaining_bytes > 0: + time_remaining = int(remaining_bytes / speed) + + self._engine.update_record('hifi', download_id, { + 'transferred': downloaded, + 'speed': speed, + 'progress': progress, + 'time_remaining': time_remaining, + }) + + def _record_to_status(self, record): + return DownloadStatus( + id=record['id'], + filename=record['filename'], + username=record['username'], + state=record['state'], + progress=record['progress'], + size=record.get('size', 0), + transferred=record.get('transferred', 0), + speed=record.get('speed', 0), + time_remaining=record.get('time_remaining'), + file_path=record.get('file_path'), + ) async def get_all_downloads(self) -> List[DownloadStatus]: - statuses = [] - with self._download_lock: - for _dl_id, info in self.active_downloads.items(): - statuses.append(DownloadStatus( - id=info['id'], - filename=info['filename'], - username=info['username'], - state=info['state'], - progress=info['progress'], - size=info['size'], - transferred=info['transferred'], - speed=info['speed'], - time_remaining=info.get('time_remaining'), - file_path=info.get('file_path'), - )) - return statuses + if self._engine is None: + return [] + return [ + self._record_to_status(record) + for record in self._engine.iter_records_for_source('hifi') + ] async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: - with self._download_lock: - info = self.active_downloads.get(download_id) - if not info: - return None - return DownloadStatus( - id=info['id'], - filename=info['filename'], - username=info['username'], - state=info['state'], - progress=info['progress'], - size=info['size'], - transferred=info['transferred'], - speed=info['speed'], - time_remaining=info.get('time_remaining'), - file_path=info.get('file_path'), - ) + if self._engine is None: + return None + record = self._engine.get_record('hifi', download_id) + return self._record_to_status(record) if record is not None else None async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool: - with self._download_lock: - if download_id not in self.active_downloads: - return False - self.active_downloads[download_id]['state'] = 'Cancelled' - if remove: - del self.active_downloads[download_id] + if self._engine is None: + return False + if self._engine.get_record('hifi', download_id) is None: + return False + self._engine.update_record('hifi', download_id, {'state': 'Cancelled'}) + if remove: + self._engine.remove_record('hifi', download_id) return True async def clear_all_completed_downloads(self) -> bool: - with self._download_lock: - to_remove = [ - did for did, info in self.active_downloads.items() - if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted') - ] - for did in to_remove: - del self.active_downloads[did] + if self._engine is None: + return True + terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} + for record in list(self._engine.iter_records_for_source('hifi')): + if record.get('state') in terminal: + self._engine.remove_record('hifi', record['id']) return True diff --git a/core/hydrabase_client.py b/core/hydrabase_client.py index f30324b2..df204d38 100644 --- a/core/hydrabase_client.py +++ b/core/hydrabase_client.py @@ -525,8 +525,8 @@ class HydrabaseClient: for album in albums: if album.image_url: return album.image_url - except Exception: - pass + except Exception as e: + logger.debug("get artist image from albums failed: %s", e) return None def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 50) -> List[Album]: diff --git a/core/imports/context.py b/core/imports/context.py index 4ed44b56..c302891e 100644 --- a/core/imports/context.py +++ b/core/imports/context.py @@ -32,6 +32,20 @@ def _first_id_value(*values: Any) -> str: return "" +def _first_source_aware_id(source: str, *values: Any) -> str: + source_name = (source or "").strip().lower() + for value in values: + if value in (None, ""): + continue + text = str(value).strip() + if not text: + continue + if source_name.startswith("spotify") and text.isdigit(): + continue + return text + return "" + + def extract_artist_name(artist: Any) -> str: if isinstance(artist, dict): return str(artist.get("name", "") or "") @@ -209,6 +223,7 @@ def get_import_has_full_metadata(context: Optional[Dict[str, Any]]) -> bool: def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: + source = get_import_source(context) track_info = get_import_track_info(context) original_search = get_import_original_search(context) search_result = get_import_search_result(context) @@ -216,7 +231,8 @@ def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: album = get_import_context_album(context) return { - "track_id": _first_id_value( + "track_id": _first_source_aware_id( + source, _first_value(track_info, "id", "track_id", "trackId", "source_track_id", default=""), _first_value(track_info, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), _first_value(original_search, "id", "track_id", "source_track_id", default=""), @@ -224,7 +240,8 @@ def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: _first_value(search_result, "id", "track_id", "source_track_id", default=""), _first_value(search_result, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), ), - "artist_id": _first_id_value( + "artist_id": _first_source_aware_id( + source, _first_value(artist, "id", "artist_id", "source_artist_id", default=""), _first_value(artist, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), _first_value(original_search, "artist_id", "source_artist_id", default=""), @@ -232,7 +249,8 @@ def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: _first_value(search_result, "artist_id", "source_artist_id", default=""), _first_value(search_result, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), ), - "album_id": _first_id_value( + "album_id": _first_source_aware_id( + source, _first_value(album, "id", "album_id", "collectionId", "source_album_id", default=""), _first_value(album, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), _first_value(original_search, "album_id", "source_album_id", default=""), @@ -257,6 +275,8 @@ def get_source_tag_names(source: str) -> Dict[str, Optional[str]]: return {"track": None, "artist": None, "album": None} if source_name == "discogs": return {"track": None, "artist": None, "album": None} + if source_name == "hifi": + return {"track": "HIFI_TRACK_ID", "artist": "HIFI_ARTIST_ID", "album": None} return {"track": None, "artist": None, "album": None} @@ -272,6 +292,8 @@ def get_library_source_id_columns(source: str) -> Dict[str, Optional[str]]: return {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"} if source_name == "discogs": return {"artist": "discogs_id", "album": "discogs_id", "track": None} + if source_name == "hifi": + return {"artist": "hifi_artist_id", "album": None, "track": "hifi_track_id"} return {} diff --git a/core/imports/file_integrity.py b/core/imports/file_integrity.py new file mode 100644 index 00000000..82ced0d1 --- /dev/null +++ b/core/imports/file_integrity.py @@ -0,0 +1,192 @@ +"""Audio file integrity checks for downloaded files. + +slskd (and other download sources) sometimes ship broken files: truncated +transfers, corrupted FLAC frames, mp3s with bad headers, or wrong files +that share a name with the target. These slip past the slskd "completed" +status and only get caught later (often by Plex/Jellyfin failing to scan +the file, or by users hearing dead air). + +Verification runs after the slskd transfer settles but before the heavy +post-processing work (tagging, copying, server sync). Failed files get +quarantined and the slot is freed for a retry from another candidate. + +Three checks, in order from cheapest to most expensive: + +1. **File-size sanity** — anything below ~10KB is almost certainly a + stub, broken transfer, or non-audio masquerading as audio. +2. **Mutagen parse** — catches truncated headers, corrupted streamheaders, + wrong-format files (mp3 with .flac extension, etc). If mutagen can't + parse the audio info block, the file won't import cleanly downstream. +3. **Duration agreement** — if the caller provides an expected duration + (Spotify/MusicBrainz `duration_ms`), the decoded length must agree + within tolerance. Catches truncated files whose headers parse fine + but whose audio is incomplete, and "wrong file" cases the slskd + transfer matched on a similarly-named track. + +This is the "tier 1" integrity layer — universal across formats, no +external binary dep. A future tier could verify the FLAC STREAMINFO MD5 +by actually decoding the audio (requires `flac` binary or libflac +wrapper); skipped for now since tier 1 catches the vast majority of +real-world corruption. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +from utils.logging_config import get_logger + + +logger = get_logger("imports.file_integrity") + +# Minimum plausible audio file size. A 1-second 64kbps mp3 is ~8KB; a +# 1-second FLAC is much larger. Anything under this is a broken stub. +_MIN_FILE_SIZE_BYTES = 10 * 1024 + +# Default tolerance for duration agreement. Most legitimate length +# variations (intro silence, encoder padding, live recording trims) sit +# inside 3 seconds. Goes up to 5s if the expected duration is itself +# long (>10 minutes) since absolute drift scales with length. +_DEFAULT_LENGTH_TOLERANCE_S = 3.0 +_LENGTH_TOLERANCE_LONG_TRACK_S = 5.0 +_LONG_TRACK_THRESHOLD_S = 600.0 # 10 minutes + + +@dataclass +class IntegrityResult: + """Outcome of an integrity check. + + `ok` is the single bit the caller cares about. `reason` is the + human-readable explanation when `ok` is False (suitable for + quarantine sidecar / log lines / UI). `checks` carries the + per-check details — useful for debugging and tests. + """ + + ok: bool + reason: str = "" + checks: Dict[str, Any] = field(default_factory=dict) + + +def check_audio_integrity( + file_path: str, + expected_duration_ms: Optional[int] = None, + *, + length_tolerance_s: Optional[float] = None, + min_file_size_bytes: int = _MIN_FILE_SIZE_BYTES, +) -> IntegrityResult: + """Verify a downloaded audio file is not broken. + + Args: + file_path: Path to the audio file on disk. + expected_duration_ms: Expected track length from the metadata + source (Spotify/MB/etc). If None, the duration check is + skipped and only the size + parse checks run. + length_tolerance_s: Override the default tolerance for the + duration check. None uses the auto-scaled default + (3s for normal tracks, 5s for >10min tracks). + min_file_size_bytes: Override the minimum size threshold. + + Returns: + IntegrityResult with `ok`, `reason`, and per-check details. + Never raises — all errors become `ok=False` with an explanatory + reason, so callers can rely on a clean boolean. + """ + import os + + checks: Dict[str, Any] = {} + + # --- Check 1: file size --- + try: + size = os.path.getsize(file_path) + except OSError as exc: + return IntegrityResult(ok=False, reason=f"Cannot stat file: {exc}", + checks={"size": "stat_failed"}) + + checks["size_bytes"] = size + if size < min_file_size_bytes: + return IntegrityResult( + ok=False, + reason=f"File too small ({size} bytes, minimum {min_file_size_bytes}) — " + "likely truncated transfer or empty stub", + checks=checks, + ) + + # --- Check 2: mutagen parse --- + try: + from mutagen import File as MutagenFile + except ImportError: + # mutagen is a hard dep elsewhere in the codebase, but degrade + # gracefully if it's somehow missing — pass with a warning + # rather than failing every download. + logger.warning("[Integrity] mutagen unavailable — skipping parse check") + checks["mutagen_parse"] = "unavailable" + return IntegrityResult(ok=True, checks=checks) + + try: + audio = MutagenFile(file_path) + except Exception as exc: + return IntegrityResult( + ok=False, + reason=f"Mutagen could not parse file: {exc}", + checks={**checks, "mutagen_parse": "exception"}, + ) + + if audio is None: + return IntegrityResult( + ok=False, + reason="Mutagen could not identify file format — likely corrupted " + "or wrong file extension", + checks={**checks, "mutagen_parse": "unidentified"}, + ) + + if audio.info is None: + return IntegrityResult( + ok=False, + reason="Mutagen parsed file but found no audio info block — " + "header damage suspected", + checks={**checks, "mutagen_parse": "no_info"}, + ) + + actual_length_s = float(getattr(audio.info, "length", 0) or 0) + checks["actual_length_s"] = actual_length_s + + if actual_length_s <= 0: + return IntegrityResult( + ok=False, + reason="Mutagen reports zero-length audio — file has no playable " + "audio data", + checks={**checks, "mutagen_parse": "zero_length"}, + ) + + # --- Check 3: duration agreement (optional) --- + if expected_duration_ms is None or expected_duration_ms <= 0: + checks["length_check"] = "skipped" + return IntegrityResult(ok=True, checks=checks) + + expected_length_s = expected_duration_ms / 1000.0 + checks["expected_length_s"] = expected_length_s + + if length_tolerance_s is None: + length_tolerance_s = ( + _LENGTH_TOLERANCE_LONG_TRACK_S + if expected_length_s > _LONG_TRACK_THRESHOLD_S + else _DEFAULT_LENGTH_TOLERANCE_S + ) + checks["length_tolerance_s"] = length_tolerance_s + + drift_s = abs(actual_length_s - expected_length_s) + checks["length_drift_s"] = drift_s + + if drift_s > length_tolerance_s: + return IntegrityResult( + ok=False, + reason=f"Duration mismatch: file is {actual_length_s:.1f}s, " + f"expected {expected_length_s:.1f}s " + f"(drift {drift_s:.1f}s > tolerance {length_tolerance_s:.1f}s) — " + "likely truncated download or wrong file matched", + checks=checks, + ) + + checks["length_check"] = "passed" + return IntegrityResult(ok=True, checks=checks) diff --git a/core/imports/file_ops.py b/core/imports/file_ops.py index f119eff5..c3620d78 100644 --- a/core/imports/file_ops.py +++ b/core/imports/file_ops.py @@ -362,8 +362,8 @@ def downsample_hires_flac(final_path, context): if os.path.exists(temp_path): try: os.remove(temp_path) - except Exception: - pass + except Exception as _e: + logger.debug("cleanup downsample temp: %s", _e) return None @@ -440,14 +440,36 @@ def create_lossy_copy(final_path): audio.save() except Exception as tag_err: logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}") + + # Honor the lossy_copy.delete_original setting — without this + # the original FLAC was always kept alongside the converted + # MP3/OPUS/AAC even when the user explicitly opted into a + # lossy-only library (Discord-reported by CAL). + if config_manager.get("lossy_copy.delete_original", False): + if os.path.normpath(out_path) != os.path.normpath(final_path): + try: + os.remove(final_path) + logger.info( + f"[Lossy Copy] Deleted original lossless source after conversion: " + f"{os.path.basename(final_path)}" + ) + except FileNotFoundError: + # Already gone — concurrent cleanup or another worker + # handled it. Not an error. + pass + except Exception as del_err: + logger.error( + f"[Lossy Copy] Could not delete original after conversion " + f"({os.path.basename(final_path)}): {del_err}" + ) return out_path logger.error(f"[Lossy Copy] ffmpeg failed: {result.stderr[:200]}") if os.path.exists(out_path): try: os.remove(out_path) - except Exception: - pass + except Exception as _e: + logger.debug("cleanup lossy copy artifact: %s", _e) return None except subprocess.TimeoutExpired: logger.warning(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}") diff --git a/core/imports/guards.py b/core/imports/guards.py index a95fbd6d..657576f0 100644 --- a/core/imports/guards.py +++ b/core/imports/guards.py @@ -82,8 +82,8 @@ def move_to_quarantine(file_path: str, context: dict, reason: str, automation_en "reason": reason or "Unknown", }, ) - except Exception: - pass + except Exception as e: + logger.debug("emit download_quarantined failed: %s", e) return str(quarantine_path) diff --git a/core/imports/paths.py b/core/imports/paths.py index 9fd9dea7..bdc12616 100644 --- a/core/imports/paths.py +++ b/core/imports/paths.py @@ -188,8 +188,8 @@ def _replace_template_variables(template: str, context: dict) -> str: resolved = resolved_client.resolve_primary_artist(itunes_artist_id) if resolved and resolved != album_artist_value: album_artist_value = resolved - except Exception: - pass + except Exception as e: + logger.debug("resolve primary artist failed: %s", e) # $cdnum — smart CD label for multi-disc filenames. Produces "CD01" / # "CD02" etc. when the album has 2+ discs, empty string otherwise. diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index a25c6be4..d7b2b7d0 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -32,6 +32,7 @@ from core.imports.context import ( get_import_track_info, normalize_import_context, ) +from core.imports.file_integrity import check_audio_integrity from core.imports.filename import extract_track_number_from_filename from core.imports.guards import check_flac_bit_depth, move_to_quarantine from core.imports.side_effects import ( @@ -143,6 +144,68 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta else: logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)") + # File integrity check: catches broken slskd transfers (truncated, + # corrupted, wrong file masquerading as the target) before we burn + # cycles on AcoustID + tagging + library sync. Universal across + # formats; failed files get quarantined and the slot freed. + try: + _normalized_for_duration = normalize_import_context(context) + _duration_track = get_import_track_info(_normalized_for_duration) + _expected_duration_ms = int(_duration_track.get("duration_ms", 0) or 0) or None + except Exception: + _expected_duration_ms = None + + try: + integrity = check_audio_integrity(file_path, _expected_duration_ms) + except Exception as integrity_error: + logger.error(f"[Integrity] Check raised unexpectedly (continuing): {integrity_error}") + integrity = None + + if integrity is not None and not integrity.ok: + logger.error(f"[Integrity] Rejected {_basename}: {integrity.reason}") + context['_integrity_failure_msg'] = integrity.reason + context['_integrity_checks'] = integrity.checks + try: + quarantine_path = move_to_quarantine( + file_path, + context, + f"Integrity check failed: {integrity.reason}", + automation_engine, + ) + logger.error(f"File quarantined due to integrity failure: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting broken file: {file_path}") + try: + os.remove(file_path) + except Exception as del_error: + logger.error(f"Could not delete broken file either: {del_error}") + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"File integrity check failed: {integrity.reason}" + ) + + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + + if integrity is not None: + logger.info( + f"[Integrity] {_basename} passed " + f"(size={integrity.checks.get('size_bytes', '?')}b, " + f"length={integrity.checks.get('actual_length_s', 0):.1f}s, " + f"drift={integrity.checks.get('length_drift_s', 'n/a')})" + ) + _skip_acoustid = False try: from core.acoustid_verification import AcoustIDVerification, VerificationResult @@ -371,8 +434,8 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") try: os.remove(file_path) - except Exception: - pass + except Exception as e: + logger.debug("delete quarantine fallback: %s", e) context['_bitdepth_rejected'] = True with matched_context_lock: @@ -499,8 +562,8 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") try: os.remove(file_path) - except Exception: - pass + except Exception as e: + logger.debug("delete quarantine fallback: %s", e) context['_bitdepth_rejected'] = True with matched_context_lock: @@ -864,6 +927,50 @@ def post_process_matched_download_with_verification(context_key, context, file_p _notify_download_completed(batch_id, task_id, success=False) return + # Integrity rejection — the inner pipeline quarantined the file + # because audio integrity (size / parse / duration) failed. Wrapper + # was previously falling through to "assuming success" because + # quarantined files have no _final_processed_path, which left the + # task showing ✅ Completed in the UI even though the file is in + # quarantine. Reported by user when downloading Mr. Morale: 3 + # tracks (Rich Interlude, Savior Interlude, Savior) showed + # Completed in the modal but were missing on disk because their + # source files failed integrity and were quarantined. + if context.get('_integrity_failure_msg'): + failure_msg = context.get('_integrity_failure_msg', 'unknown') + logger.error( + f"Task {task_id} failed integrity check — marking failed: {failure_msg}" + ) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"File integrity check failed: {failure_msg}" + ) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + # Race guard failure — inner code set this when the source file + # disappeared and there was no known destination to fall back on + # (vs the legitimate race-guard skip where a sibling thread + # already moved the file to its destination). + if context.get('_race_guard_failed'): + logger.error(f"Task {task_id} failed race guard — source file gone with no known destination") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + "Source file disappeared before post-processing could complete" + ) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + expected_final_path = context.get('_final_processed_path') if not expected_final_path: logger.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success") diff --git a/core/imports/resolution.py b/core/imports/resolution.py index 89a016d2..e37b3d3a 100644 --- a/core/imports/resolution.py +++ b/core/imports/resolution.py @@ -2,12 +2,30 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional from core.metadata import registry as metadata_registry +from core.metadata.types import Album from utils.logging_config import get_logger +# Per-source typed converter dispatch — same registry pattern as +# ``core/metadata/album_tracks.py``. When the embedded ``album`` blob in +# a track response is dispatched through the typed converter for that +# provider, the resulting Album fields drive the album_payload below. +# Falls through to the legacy duck-typed path when source is empty, +# unknown, or the converter raises. +_TYPED_ALBUM_CONVERTERS: Dict[str, Callable[[Dict[str, Any]], Album]] = { + 'spotify': Album.from_spotify_dict, + 'itunes': Album.from_itunes_dict, + 'deezer': Album.from_deezer_dict, + 'discogs': Album.from_discogs_dict, + 'musicbrainz': Album.from_musicbrainz_dict, + 'hydrabase': Album.from_hydrabase_dict, + 'qobuz': Album.from_qobuz_dict, +} + + logger = get_logger("imports.resolution") @@ -156,17 +174,62 @@ def _build_single_import_context_payload( album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[])) if isinstance(album_data, dict): - album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name - album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) - release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) - album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) - total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) - album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] - if not album_image_url: - album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') - if not album_image_url and album_images: - album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') - album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + # Typed dispatch: when the source is a known provider, route the + # embedded album blob through Album.from__dict() and read + # canonical fields off the typed result. Falls back to the + # legacy duck-typed extraction below on unknown/missing source + # OR if the converter raises (so a converter bug can't break + # import context resolution). + typed_album: Optional[Album] = None + source_key = (source or '').strip().lower() + if source_key: + converter = _TYPED_ALBUM_CONVERTERS.get(source_key) + if converter is not None: + try: + typed_album = converter(album_data) + except Exception as exc: + logger.debug( + "Typed album converter failed for source %s in import " + "context build, falling back to legacy: %s", source, exc, + ) + typed_album = None + + if typed_album is not None: + if typed_album.name: + album_name = typed_album.name + if typed_album.id: + album_id = typed_album.id + if typed_album.release_date: + release_date = typed_album.release_date + if typed_album.album_type: + album_type = typed_album.album_type + if typed_album.total_tracks: + total_tracks = typed_album.total_tracks + # Preserve raw images list verbatim (legacy behavior — some + # downstream consumers iterate the full multi-resolution + # array to pick a different size). + raw_images = album_data.get('images') + if isinstance(raw_images, list) and raw_images: + album_images = raw_images + if not album_image_url: + album_image_url = typed_album.image_url or '' + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + album_artists = _normalize_context_artists( + [{'name': name} for name in typed_album.artists] + ) + else: + album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name + album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) + release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) + album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) + total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) + album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] + if not album_image_url: + album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) elif album_data: album_name = album_name or str(album_data) @@ -344,8 +407,8 @@ def get_single_track_import_context( 'genres', default=[], ) or [] - except Exception: - pass + except Exception as e: + logger.debug("override artist genres: %s", e) return payload except Exception as exc: logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc) @@ -398,8 +461,8 @@ def get_single_track_import_context( 'genres', default=[], ) or [] - except Exception: - pass + except Exception as e: + logger.debug("artist genres lookup: %s", e) return payload return _build_single_import_fallback_context(title, artist, source_priority) diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index 2824dd2c..6252abd7 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -72,8 +72,8 @@ def emit_track_downloaded(context: Dict[str, Any], automation_engine=None) -> No "quality": context.get("_audio_quality", "Unknown"), }, ) - except Exception: - pass + except Exception as e: + logger.debug("track_downloaded emit failed: %s", e) def record_library_history_download(context: Dict[str, Any]) -> None: @@ -88,6 +88,7 @@ def record_library_history_download(context: Dict[str, Any]) -> None: "hifi": "HiFi", "deezer_dl": "Deezer", "lidarr": "Lidarr", + "soundcloud": "SoundCloud", } download_source = source_map.get(username, "Soulseek") @@ -119,7 +120,7 @@ def record_library_history_download(context: Dict[str, Any]) -> None: source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "") source_track_title = search_result.get("title", "") or search_result.get("name", "") source_artist = search_result.get("artist", "") - if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr"): + if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr", "soundcloud"): stream_id = source_filename.split("||")[0] if stream_id and not source_track_id: source_track_id = stream_id @@ -142,8 +143,8 @@ def record_library_history_download(context: Dict[str, Any]) -> None: acoustid_result=acoustid_result, source_artist=source_artist, ) - except Exception: - pass + except Exception as e: + logger.debug("library history record failed: %s", e) def record_download_provenance(context: Dict[str, Any]) -> None: @@ -159,6 +160,7 @@ def record_download_provenance(context: Dict[str, Any]) -> None: "hifi": "hifi", "deezer_dl": "deezer", "lidarr": "lidarr", + "soundcloud": "soundcloud", }.get(username, "soulseek") ti = context.get("track_info") or context.get("search_result") or {} @@ -186,8 +188,32 @@ def record_download_provenance(context: Dict[str, Any]) -> None: sample_rate = getattr(audio.info, "sample_rate", None) bitrate = getattr(audio.info, "bitrate", None) bit_depth = getattr(audio.info, "bits_per_sample", None) - except Exception: - pass + except Exception as e: + logger.debug("audio info probe failed: %s", e) + + # Pull the metadata-source IDs out of context. ``embed_source_ids`` + # in core/metadata/source.py wrote them to ``_embedded_id_tags`` + # at the end of post-processing — we persist them here so the + # watchlist scanner can recognize freshly downloaded files + # without waiting for the async enrichment workers. + embedded = context.get("_embedded_id_tags") or {} + + def _embedded(*keys): + for k in keys: + v = embedded.get(k) + if v: + return str(v) + return None + + spotify_track_id = _embedded("SPOTIFY_TRACK_ID") + itunes_track_id = _embedded("ITUNES_TRACK_ID") + deezer_track_id = _embedded("DEEZER_TRACK_ID") + tidal_track_id = _embedded("TIDAL_TRACK_ID") + qobuz_track_id = _embedded("QOBUZ_TRACK_ID") + musicbrainz_recording_id = _embedded("MUSICBRAINZ_RECORDING_ID") + audiodb_id = _embedded("AUDIODB_TRACK_ID") + soul_id = _embedded("SOUL_ID") + isrc = context.get("_isrc") db = get_database() db.record_track_download( @@ -203,9 +229,18 @@ def record_download_provenance(context: Dict[str, Any]) -> None: bit_depth=bit_depth, sample_rate=sample_rate, bitrate=bitrate, + spotify_track_id=spotify_track_id, + itunes_track_id=itunes_track_id, + deezer_track_id=deezer_track_id, + tidal_track_id=tidal_track_id, + qobuz_track_id=qobuz_track_id, + musicbrainz_recording_id=musicbrainz_recording_id, + audiodb_id=audiodb_id, + soul_id=soul_id, + isrc=isrc, ) - except Exception: - pass + except Exception as e: + logger.debug("record_download_provenance failed: %s", e) def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any]) -> None: @@ -286,7 +321,18 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[ audio = MutagenFile(final_path) if audio and hasattr(audio, "info") and audio.info and hasattr(audio.info, "bitrate"): bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0 - except Exception: + except Exception as e: + logger.debug("bitrate read failed: %s", e) + + # File size on disk (powers Library Disk Usage card on Stats). + # SoulSync standalone is the only path where the file is local + # at insert time, so we read it directly via os.path.getsize. + # Mirrors what JellyfinTrack/NavidromeTrack pull from API + # responses for the media-server flows. + file_size = None + try: + file_size = os.path.getsize(final_path) or None + except OSError: pass artist_id = _stable_soulsync_id(artist_name.lower().strip()) @@ -325,8 +371,8 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[ f"UPDATE artists SET {artist_source_col} = ? WHERE id = ?", (artist_source_id, artist_id), ) - except Exception: - pass + except Exception as e: + logger.debug("artist source-id update failed: %s", e) cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,)) if not cursor.fetchone(): @@ -356,8 +402,8 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[ f"UPDATE albums SET {album_source_col} = ? WHERE id = ?", (album_source_id, album_id), ) - except Exception: - pass + except Exception as e: + logger.debug("album source-id update failed: %s", e) track_artist = None track_artists_list = track_info.get("artists", []) or original_search.get("artists", []) @@ -375,9 +421,9 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[ cursor.execute( """ INSERT INTO tracks (id, album_id, artist_id, title, track_number, - duration, file_path, bitrate, track_artist, server_source, + duration, file_path, bitrate, file_size, track_artist, server_source, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, ( track_id, @@ -388,6 +434,7 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[ duration_ms, final_path, bitrate, + file_size, track_artist, ), ) @@ -404,8 +451,8 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[ f"UPDATE tracks SET {track_album_col} = ? WHERE id = ?", (album_source_id, track_id), ) - except Exception: - pass + except Exception as e: + logger.debug("track source-id update failed: %s", e) conn.commit() logger.info("[SoulSync Library] Added: %s / %s / %s", artist_name, album_name, track_name) diff --git a/core/itunes_client.py b/core/itunes_client.py index a62fb035..95d64669 100644 --- a/core/itunes_client.py +++ b/core/itunes_client.py @@ -380,8 +380,8 @@ class iTunesClient: for raw in cached_results: try: tracks.append(Track.from_itunes_track(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Track.from_itunes_track cache parse: %s", e) if tracks: return tracks @@ -569,8 +569,8 @@ class iTunesClient: for raw in cached_results: try: albums.append(Album.from_itunes_album(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Album.from_itunes_album cache parse: %s", e) if albums: return albums @@ -893,8 +893,8 @@ class iTunesClient: for item in results: if item.get('wrapperType') == 'artist' and item.get('artistName'): return item['artistName'] - except Exception: - pass + except Exception as e: + logger.debug("itunes lookup artistId %s: %s", artist_id, e) return None def search_artists(self, query: str, limit: int = 20) -> List[Artist]: @@ -911,8 +911,8 @@ class iTunesClient: for raw in cached_results: try: artists.append(Artist.from_itunes_artist(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Artist.from_itunes_artist cache parse: %s", e) if artists: return artists diff --git a/core/itunes_worker.py b/core/itunes_worker.py index 1e1c2e9d..50cf0df0 100644 --- a/core/itunes_worker.py +++ b/core/itunes_worker.py @@ -3,12 +3,14 @@ import re import threading import time from difflib import SequenceMatcher +from types import SimpleNamespace from typing import Optional, Dict, Any, List from datetime import datetime, timedelta from utils.logging_config import get_logger from database.music_database import MusicDatabase from core.itunes_client import iTunesClient from core.worker_utils import interruptible_sleep, set_album_api_track_count +from core.enrichment.manual_match_honoring import honor_stored_match logger = get_logger("itunes_worker") @@ -142,8 +144,8 @@ class iTunesWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null id table resolve failed: %s", e) continue self._process_item(item) @@ -526,11 +528,49 @@ class iTunesWorker: # ── Individual fallback processing ───────────────────────────────── + def _refresh_album_via_stored_id(self, album_id, stored_id, api_album_dict): + """Issue #501 callback. Convert ``client.get_album()`` dict into + the Album-shaped object ``_update_album`` expects, then call it. + Preserves the manual match — never overwrites the stored ID + with a different name-search result.""" + images = api_album_dict.get('images') or [] + image_url = '' + if images and isinstance(images[0], dict): + image_url = images[0].get('url', '') or '' + adapter = SimpleNamespace( + id=api_album_dict.get('id') or stored_id, + name=api_album_dict.get('name', ''), + image_url=image_url, + album_type=api_album_dict.get('album_type', 'album'), + release_date=api_album_dict.get('release_date', ''), + total_tracks=api_album_dict.get('total_tracks', 0), + ) + self._update_album(album_id, adapter) + + def _refresh_track_via_stored_id(self, track_id, stored_id, api_track_dict): + """Track-level callback — track update only writes ID + status, + no metadata backfill, so the dict shape is irrelevant beyond + carrying the stored ID through.""" + adapter = SimpleNamespace(id=api_track_dict.get('id') or stored_id) + self._update_track_from_search(track_id, adapter) + def _process_album_individual(self, item: Dict[str, Any]): album_id = item['id'] album_name = item['name'] artist_name = item.get('artist', '') + # Issue #501: honor manual matches (see SpotifyWorker for full + # explanation — same pattern across every per-source worker). + if honor_stored_match( + db=self.db, entity_table='albums', entity_id=album_id, + id_column='itunes_album_id', + client_fetch_fn=self.client.get_album, + on_match_fn=self._refresh_album_via_stored_id, + log_prefix='iTunes', + ): + self.stats['matched'] += 1 + return + query = f"{artist_name} {album_name}" if artist_name else album_name results = self.client.search_albums(query, limit=5) @@ -561,6 +601,17 @@ class iTunesWorker: track_name = item['name'] artist_name = item.get('artist', '') + # Issue #501: honor manual matches. + if honor_stored_match( + db=self.db, entity_table='tracks', entity_id=track_id, + id_column='itunes_track_id', + client_fetch_fn=self.client.get_track_details, + on_match_fn=self._refresh_track_via_stored_id, + log_prefix='iTunes', + ): + self.stats['matched'] += 1 + return + query = f"{artist_name} {track_name}" if artist_name else track_name results = self.client.search_tracks(query, limit=5) diff --git a/core/jellyfin_client.py b/core/jellyfin_client.py index 4cf1500e..1257d4ab 100644 --- a/core/jellyfin_client.py +++ b/core/jellyfin_client.py @@ -1,33 +1,19 @@ import requests import time from typing import List, Optional, Dict, Any -from dataclasses import dataclass from datetime import datetime import json from utils.logging_config import get_logger from config.settings import config_manager +# Shared dataclasses live in the neutral media_server package — every +# server client used to define a near-identical XTrackInfo / +# XPlaylistInfo. Lifted to one canonical type so consumers (matching +# engine, sync service) get a single import. +from core.media_server.types import TrackInfo, PlaylistInfo + logger = get_logger("jellyfin_client") -@dataclass -class JellyfinTrackInfo: - id: str - title: str - artist: str - album: str - duration: int - track_number: Optional[int] = None - year: Optional[int] = None - rating: Optional[float] = None - -@dataclass -class JellyfinPlaylistInfo: - id: str - title: str - description: Optional[str] - duration: int - leaf_count: int - tracks: List[JellyfinTrackInfo] class JellyfinArtist: """Wrapper class to mimic Plex artist object interface""" @@ -116,9 +102,15 @@ class JellyfinTrack: # File path and media info (used by quality scanner and DB update) self.path = jellyfin_data.get('Path') - # Extract bitrate from MediaSources if available + # Extract bitrate + file size from MediaSources if available. + # `file_size` powers the Library Disk Usage card on the Stats + # page — populated free during the deep scan from data Jellyfin + # already returns in MediaSources[]. media_sources = jellyfin_data.get('MediaSources', []) self.bitRate = (media_sources[0].get('Bitrate') or 0) // 1000 if media_sources else None # Convert bps to kbps + self.file_size = (media_sources[0].get('Size') or 0) if media_sources else None + if self.file_size == 0: + self.file_size = None def _parse_date(self, date_str: Optional[str]) -> Optional[datetime]: if not date_str: @@ -140,7 +132,10 @@ class JellyfinTrack: return self._client.get_album_by_id(self._album_id) return None -class JellyfinClient: +from core.media_server.contract import MediaServerClient + + +class JellyfinClient(MediaServerClient): def __init__(self): self.base_url: Optional[str] = None self.api_key: Optional[str] = None @@ -1195,7 +1190,7 @@ class JellyfinClient: return stats - def get_all_playlists(self) -> List[JellyfinPlaylistInfo]: + def get_all_playlists(self) -> List[PlaylistInfo]: """Get all playlists from Jellyfin server""" if not self.ensure_connection(): return [] @@ -1212,7 +1207,7 @@ class JellyfinClient: playlists = [] for item in response.get('Items', []): - playlist_info = JellyfinPlaylistInfo( + playlist_info = PlaylistInfo( id=item.get('Id', ''), title=item.get('Name', 'Unknown Playlist'), description=item.get('Overview'), @@ -1229,7 +1224,7 @@ class JellyfinClient: logger.error(f"Error getting playlists from Jellyfin: {e}") return [] - def get_playlist_by_name(self, name: str) -> Optional[JellyfinPlaylistInfo]: + def get_playlist_by_name(self, name: str) -> Optional[PlaylistInfo]: """Get a specific playlist by name""" playlists = self.get_all_playlists() for playlist in playlists: @@ -1446,8 +1441,8 @@ class JellyfinClient: response = requests.delete(url, headers=headers, timeout=10) if response.status_code in [200, 204]: logger.info(f"Deleted existing backup playlist '{target_name}'") - except Exception: - pass # Target doesn't exist, which is fine + except Exception as e: + logger.debug("backup playlist precheck: %s", e) # Create new playlist with copied tracks try: diff --git a/core/lastfm_worker.py b/core/lastfm_worker.py index 2509f4aa..69a4c360 100644 --- a/core/lastfm_worker.py +++ b/core/lastfm_worker.py @@ -155,8 +155,8 @@ class LastFMWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null id table resolve failed: %s", e) continue self._process_item(item) diff --git a/core/library/duplicate_cleaner.py b/core/library/duplicate_cleaner.py index 1b484caf..29fe4729 100644 --- a/core/library/duplicate_cleaner.py +++ b/core/library/duplicate_cleaner.py @@ -209,8 +209,8 @@ def _run_duplicate_cleaner(): 'duplicates_found': str(duplicates_found), 'space_freed': f"{space_mb:.1f} MB", }) - except Exception: - pass + except Exception as e: + logger.debug("emit duplicate_scan_completed failed: %s", e) except Exception as e: logger.error(f"[Duplicate Cleaner] Critical error: {e}") diff --git a/core/library/path_resolver.py b/core/library/path_resolver.py new file mode 100644 index 00000000..44a157d8 --- /dev/null +++ b/core/library/path_resolver.py @@ -0,0 +1,173 @@ +"""Resolve database-stored file paths to actual files on disk. + +Database track rows store file paths as the media server reported them +(`/music/Artist/Album/track.flac`, `H:\\Music\\Artist\\...`, etc). When +SoulSync runs in Docker, those paths don't exist as-is inside the +container — the user's library is bind-mounted at a container path +(commonly `/music`) that has nothing to do with what Plex/Jellyfin +recorded. Same problem for native installs that point at a NAS via SMB: +the path the media server scanned isn't the path SoulSync reads. + +The resolver tries the raw path first (cheap happy-path), then walks +progressively shorter suffixes against every configured base directory: +the transfer folder, the slskd download folder, every configured Plex +library location, and every entry in the user's `library.music_paths` +config. The first existing match wins. + +This module replaces four duplicated copies of the same function (each +with the same incomplete logic) that lived in +`core/repair_worker.py` and three modules under `core/repair_jobs/`. +The duplicates only checked the transfer + download folders and +silently returned None for files in the actual media-server library — +which is why, for example, the Album Completeness "Auto-Fill" button +returned ``Could not determine album folder from existing tracks`` for +every Docker user (issue #476). + +The web server has its own near-duplicate at +``web_server.py:_resolve_library_file_path`` which already covers the +full search space; this module is the lifted, shared version usable +from any background worker. +""" + +from __future__ import annotations + +import os +from typing import Any, Iterable, Optional + +from utils.logging_config import get_logger + + +logger = get_logger("library.path_resolver") + + +def _docker_resolve_path(path_str: Any) -> Optional[str]: + """Translate Windows-style paths to the Docker container layout. + + Mirrors ``core/imports/paths.docker_resolve_path`` but kept local to + avoid a cross-package import in case this module is consumed early + in a job startup. Returns the input unchanged outside Docker. + """ + if not isinstance(path_str, str): + return None + if ( + os.path.exists("/.dockerenv") + and len(path_str) >= 3 + and path_str[1] == ":" + and path_str[0].isalpha() + ): + drive_letter = path_str[0].lower() + rest = path_str[2:].replace("\\", "/") + return f"/host/mnt/{drive_letter}{rest}" + return path_str + + +def _collect_base_dirs( + transfer_folder: Optional[str], + download_folder: Optional[str], + config_manager: Any, + plex_client: Any, +) -> list[str]: + """Build the ordered list of base directories to probe.""" + candidates: list[Optional[str]] = [] + + if transfer_folder: + candidates.append(_docker_resolve_path(transfer_folder)) + if download_folder: + candidates.append(_docker_resolve_path(download_folder)) + + if config_manager is not None: + try: + transfer_cfg = config_manager.get("soulseek.transfer_path", "") or "" + download_cfg = config_manager.get("soulseek.download_path", "") or "" + if transfer_cfg: + candidates.append(_docker_resolve_path(transfer_cfg)) + if download_cfg: + candidates.append(_docker_resolve_path(download_cfg)) + except Exception as e: + logger.debug("soulseek paths read failed: %s", e) + + # Plex-reported library locations (handles "Plex scanned at /music but + # SoulSync mounts at /library" cases). + if plex_client is not None: + try: + server = getattr(plex_client, "server", None) + music_library = getattr(plex_client, "music_library", None) + if server is not None and music_library is not None: + for loc in getattr(music_library, "locations", []) or []: + if loc: + candidates.append(loc) + except Exception as e: + logger.debug("plex locations read failed: %s", e) + + # User-configured library music paths (Settings → Library → Music Paths). + if config_manager is not None: + try: + music_paths = config_manager.get("library.music_paths", []) or [] + if isinstance(music_paths, Iterable): + for p in music_paths: + if isinstance(p, str) and p.strip(): + candidates.append(_docker_resolve_path(p.strip())) + except Exception as e: + logger.debug("music paths read failed: %s", e) + + # De-duplicate while preserving order, drop empties / non-existent dirs. + seen: set[str] = set() + out: list[str] = [] + for c in candidates: + if not c or c in seen: + continue + seen.add(c) + if os.path.isdir(c): + out.append(c) + return out + + +def resolve_library_file_path( + file_path: Any, + *, + transfer_folder: Optional[str] = None, + download_folder: Optional[str] = None, + config_manager: Any = None, + plex_client: Any = None, +) -> Optional[str]: + """Resolve a stored DB path to an actual file on disk. + + Args: + file_path: The path as recorded in the database (may not exist + as-is in the current process's filesystem view). + transfer_folder: Optional explicit transfer-folder override + (bypasses the config_manager lookup). Useful when the caller + already cached one. + download_folder: Optional explicit download-folder override. + config_manager: When provided, the resolver also pulls + ``soulseek.transfer_path``, ``soulseek.download_path``, and + ``library.music_paths`` from config to expand the search. + plex_client: When provided, every Plex-reported music-library + location is added to the search. + + Returns: + The first existing path on disk, or None when no match is found. + Never raises — failure is the None return. + """ + if not isinstance(file_path, str) or not file_path: + return None + + if os.path.exists(file_path): + return file_path + + path_parts = file_path.replace("\\", "/").split("/") + base_dirs = _collect_base_dirs(transfer_folder, download_folder, config_manager, plex_client) + if not base_dirs: + return None + + # Skip index 0 to avoid drive-letter / leading-slash artifacts + # (e.g. "E:" or "" from a leading "/"). + for base in base_dirs: + for i in range(1, len(path_parts)): + candidate = os.path.join(base, *path_parts[i:]) + if os.path.exists(candidate): + return candidate + return None + + +__all__ = ["resolve_library_file_path"] diff --git a/core/library/redownload.py b/core/library/redownload.py index cf1bb182..08d5cd70 100644 --- a/core/library/redownload.py +++ b/core/library/redownload.py @@ -208,7 +208,7 @@ def redownload_start(track_id): # Build a TrackResult-like candidate and submit to download def _run_redownload(): try: - from core.soulseek_client import TrackResult + from core.download_plugins.types import TrackResult from core.itunes_client import Track as MetaTrack tr = TrackResult( username=candidate['username'], diff --git a/core/library/retag.py b/core/library/retag.py index 82cd949f..48be9800 100644 --- a/core/library/retag.py +++ b/core/library/retag.py @@ -265,8 +265,8 @@ def execute_retag(group_id, album_id, deps: RetagDeps): try: os.remove(old_cover) logger.warning("[Retag] Removed orphaned cover.jpg from old directory") - except Exception: - pass + except Exception as e: + logger.debug("remove orphaned cover failed: %s", e) # Cleanup old empty directories transfer_dir = deps.docker_resolve_path(deps.config_manager.get('soulseek.transfer_path', './Transfer')) diff --git a/core/library/track_identity.py b/core/library/track_identity.py new file mode 100644 index 00000000..2b43650d --- /dev/null +++ b/core/library/track_identity.py @@ -0,0 +1,300 @@ +"""Match a metadata-source track against the library by stable external IDs. + +Discord-reported (CAL): the watchlist scanner re-downloaded a track that +already existed on disk because the library DB had stale album metadata +(track tagged on album "Left Alone" while Spotify reported it as on the +"NPC" single). The matching logic relied on title + artist + album fuzzy +comparison; the album fuzzy correctly said the names didn't match, the +scanner declared the track missing, and the wishlist re-added + re- +downloaded it on every scan. + +The track has a stable external identity though — every download embeds +Spotify / iTunes / Deezer / Tidal / Qobuz / MusicBrainz / AudioDB / +Hydrabase / ISRC IDs as both file tags AND DB columns. This module pulls +those IDs off either side and asks: do we already have a row in the +``tracks`` table whose external-ID column matches one of the source +track's IDs? If yes, the track is NOT missing, regardless of how the +album metadata drifted between sources. + +Provider-neutral by design — no spotify-only paths. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from utils.logging_config import get_logger + +logger = get_logger("library.track_identity") + + +# Maps the conceptual ID name (used in the source-track dict we extract +# below) to the column name on the library ``tracks`` table where that +# ID is persisted. Keep the column names in sync with the schema in +# ``database/music_database.py``. +EXTERNAL_ID_COLUMNS: Dict[str, str] = { + 'spotify_id': 'spotify_track_id', + 'itunes_id': 'itunes_track_id', + 'deezer_id': 'deezer_id', + 'tidal_id': 'tidal_id', + 'qobuz_id': 'qobuz_id', + 'mbid': 'musicbrainz_recording_id', + 'audiodb_id': 'audiodb_id', + 'soul_id': 'soul_id', + 'isrc': 'isrc', +} + + +def _coerce(value: Any) -> Optional[str]: + """Return value as a non-empty string, or None for empty / missing.""" + if value is None: + return None + text = str(value).strip() + return text or None + + +def _get(track: Any, *names: str) -> Optional[str]: + """Read the first non-empty attribute / dict key from ``names`` off + ``track``. Accepts both dict-style and dataclass / object tracks.""" + for name in names: + try: + value = track[name] if isinstance(track, dict) else getattr(track, name, None) + except (TypeError, KeyError): + value = None + coerced = _coerce(value) + if coerced is not None: + return coerced + return None + + +def extract_external_ids(track: Any, source_hint: Optional[str] = None) -> Dict[str, str]: + """Pull every recognized external ID off a metadata-source track. + + Handles the source-source naming drift: Spotify tracks expose ``id`` + as the Spotify track ID; Deezer tracks expose ``id`` as the Deezer + track ID; iTunes tracks may use ``trackId`` or ``id``. The disamb- + iguating field is ``provider`` / ``source`` / ``_source``. Tracks + coming from a SoulSync internal pipeline often carry every known ID + set to its source-specific value — we just collect whatever's there. + + ``source_hint`` is the caller's known answer to "where did this + track dict come from?" — used as a fallback when the track itself + doesn't carry a provider / source / _source field. Spotify and + iTunes return raw API responses without provider tags, so the + watchlist scanner passes ``get_primary_source()`` here to make sure + a Spotify-primary scan isn't silently no-opping just because the + raw API track has no provider key. + + Returns a dict mapping conceptual ID name → ID value. Keys present + in ``EXTERNAL_ID_COLUMNS``. Empty dict when no IDs are available. + """ + if track is None: + return {} + + ids: Dict[str, str] = {} + + # Provider-neutral fields that carry their own name regardless of + # source. Most internal SoulSync tracks have these set; external + # source responses usually only have one of them populated. + direct_id_fields = { + 'spotify_id': ('spotify_id', 'spotify_track_id', 'SPOTIFY_TRACK_ID'), + 'itunes_id': ('itunes_id', 'itunes_track_id', 'trackId', 'ITUNES_TRACK_ID'), + 'deezer_id': ('deezer_id', 'deezer_track_id', 'DEEZER_TRACK_ID'), + 'tidal_id': ('tidal_id', 'tidal_track_id', 'TIDAL_TRACK_ID'), + 'qobuz_id': ('qobuz_id', 'qobuz_track_id', 'QOBUZ_TRACK_ID'), + 'mbid': ('musicbrainz_recording_id', 'mbid', 'MUSICBRAINZ_RECORDING_ID'), + 'audiodb_id': ('audiodb_id', 'idTrack', 'AUDIODB_TRACK_ID'), + 'soul_id': ('soul_id', 'SOUL_ID'), + 'isrc': ('isrc', 'ISRC'), + } + for name, candidates in direct_id_fields.items(): + value = _get(track, *candidates) + if value: + ids[name] = value + + # Provider field tells us which native ``id`` belongs to. Without + # this, a Deezer track's ``id`` field would be silently ignored + # (we wouldn't know to map it to deezer_id). Convention varies by + # client: Spotify-shaped tracks usually have no provider field, + # Deezer / Discogs / Hydrabase clients tag tracks with ``_source``, + # internal pipeline normalization may use ``source`` or ``provider``. + # Fall back to the caller's source_hint when the track has no + # provider field of its own (Spotify / iTunes raw API responses). + provider = (_get(track, 'provider', 'source', '_source') or source_hint or '').lower() + native_id = _get(track, 'id') + if native_id and provider: + provider_to_key = { + 'spotify': 'spotify_id', + 'itunes': 'itunes_id', + 'deezer': 'deezer_id', + 'tidal': 'tidal_id', + 'qobuz': 'qobuz_id', + 'musicbrainz': 'mbid', + 'audiodb': 'audiodb_id', + 'hydrabase': 'soul_id', + } + key = provider_to_key.get(provider) + if key and key not in ids: + ids[key] = native_id + + return ids + + +def find_library_track_by_external_id( + db: Any, + *, + external_ids: Dict[str, str], + server_source: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Return a row from the ``tracks`` table whose any external ID + column matches one of the provided IDs, or None if no match. + + Returns a sqlite3.Row-like dict so callers can read whatever fields + they want (id, title, file_path, etc.). When ``server_source`` is + set, restrict matches to tracks scanned from that media server — + avoids false positives when a user binds the same DB into multiple + profiles/servers. + + Performance: every external_id column is indexed in the schema, so + each OR clause hits an index. Limit 1 because we only need to know + whether a match exists. + """ + if not external_ids: + return None + + clauses: List[str] = [] + params: List[Any] = [] + for id_name, id_value in external_ids.items(): + column = EXTERNAL_ID_COLUMNS.get(id_name) + if not column or not id_value: + continue + clauses.append(f"({column} = ? AND {column} IS NOT NULL AND {column} != '')") + params.append(id_value) + + if not clauses: + return None + + where_external = " OR ".join(clauses) + + # Optional server_source filter + if server_source: + sql = ( + f"SELECT * FROM tracks WHERE ({where_external}) " + f"AND (server_source = ? OR server_source IS NULL) LIMIT 1" + ) + params.append(server_source) + else: + sql = f"SELECT * FROM tracks WHERE ({where_external}) LIMIT 1" + + conn = None + try: + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute(sql, params) + row = cursor.fetchone() + if row is None: + return None + # sqlite3.Row supports keys() — return as dict for caller stability. + try: + return dict(row) + except (TypeError, ValueError): + # Fallback for cursors that don't return Row objects. + cols = [c[0] for c in cursor.description] + return dict(zip(cols, row, strict=False)) + except Exception as exc: + logger.debug(f"find_library_track_by_external_id query failed: {exc}") + return None + finally: + if conn is not None: + try: + conn.close() + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down + pass + + +# Maps the conceptual ID name to the column on the ``track_downloads`` +# (provenance) table where SoulSync persists the IDs at download time. +# Naming convention differs from ``tracks``: provenance uses the +# explicit ``_track_id`` suffix to match the existing column shape. +PROVENANCE_ID_COLUMNS: Dict[str, str] = { + 'spotify_id': 'spotify_track_id', + 'itunes_id': 'itunes_track_id', + 'deezer_id': 'deezer_track_id', + 'tidal_id': 'tidal_track_id', + 'qobuz_id': 'qobuz_track_id', + 'mbid': 'musicbrainz_recording_id', + 'audiodb_id': 'audiodb_id', + 'soul_id': 'soul_id', + 'isrc': 'isrc', +} + + +def find_provenance_by_external_id( + db: Any, + *, + external_ids: Dict[str, str], +) -> Optional[Dict[str, Any]]: + """Return a row from the ``track_downloads`` table whose any external + ID column matches one of the provided IDs, or None. + + Used as a second-tier fallback by the watchlist scanner: when the + primary library tracks-table lookup misses (e.g. the row exists but + the enrichment worker hasn't backfilled its IDs yet, or the row + doesn't exist yet because the media-server scan hasn't run since the + download), this checks whether SoulSync downloaded the file recently + enough that the IDs are sitting in the provenance table. + + Caller should typically also confirm the ``file_path`` on the + returned row still exists on disk before treating the track as + "already in library" — otherwise a deleted file would prevent + re-download. + """ + if not external_ids: + return None + + clauses: List[str] = [] + params: List[Any] = [] + for id_name, id_value in external_ids.items(): + column = PROVENANCE_ID_COLUMNS.get(id_name) + if not column or not id_value: + continue + clauses.append(f"({column} = ? AND {column} IS NOT NULL AND {column} != '')") + params.append(id_value) + + if not clauses: + return None + + where_external = " OR ".join(clauses) + sql = f"SELECT * FROM track_downloads WHERE ({where_external}) ORDER BY id DESC LIMIT 1" + + conn = None + try: + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute(sql, params) + row = cursor.fetchone() + if row is None: + return None + try: + return dict(row) + except (TypeError, ValueError): + cols = [c[0] for c in cursor.description] + return dict(zip(cols, row, strict=False)) + except Exception as exc: + logger.debug(f"find_provenance_by_external_id query failed: {exc}") + return None + finally: + if conn is not None: + try: + conn.close() + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down + pass + + +__all__ = [ + 'EXTERNAL_ID_COLUMNS', + 'PROVENANCE_ID_COLUMNS', + 'extract_external_ids', + 'find_library_track_by_external_id', + 'find_provenance_by_external_id', +] diff --git a/core/library_reorganize.py b/core/library_reorganize.py index 6d62da98..a0fb86f4 100644 --- a/core/library_reorganize.py +++ b/core/library_reorganize.py @@ -506,7 +506,7 @@ def load_album_and_tracks(db, album_id): if conn is not None: try: conn.close() - except Exception: + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down pass @@ -938,8 +938,8 @@ class _RunContext: return try: self.on_progress(updates) - except Exception: - pass + except Exception as e: + logger.debug("progress emit failed: %s", e) def record_error(self, track_id, title, message, kind: str = 'skipped') -> None: with self.state_lock: @@ -1185,8 +1185,8 @@ def reorganize_album( return try: on_progress(updates) - except Exception: - pass + except Exception as e: + logger.debug("reorganize progress callback failed: %s", e) # Load album + tracks album_data, tracks = load_album_and_tracks(db, album_id) @@ -1329,7 +1329,7 @@ def reorganize_album( try: if os.path.isdir(staging_album_dir): shutil.rmtree(staging_album_dir, ignore_errors=True) - except Exception: + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down pass # Best-effort cleanup of source directories. For each touched dir @@ -1343,14 +1343,14 @@ def reorganize_album( if _has_remaining_audio(src_dir): continue _delete_album_sidecars(src_dir) - except Exception: + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down pass if cleanup_empty_dir_fn: for src_dir in src_dirs_touched: try: cleanup_empty_dir_fn(src_dir) - except Exception: + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down pass # Prune empty *destination* siblings — e.g. when a previous diff --git a/core/lidarr_download_client.py b/core/lidarr_download_client.py index f6a640f3..52bee817 100644 --- a/core/lidarr_download_client.py +++ b/core/lidarr_download_client.py @@ -28,12 +28,15 @@ from utils.logging_config import get_logger from config.settings import config_manager # Import Soulseek data structures for drop-in replacement compatibility -from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus +from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus logger = get_logger("lidarr_client") -class LidarrDownloadClient: +from core.download_plugins.base import DownloadSourcePlugin + + +class LidarrDownloadClient(DownloadSourcePlugin): """Lidarr download client — uses Lidarr as a download source for Usenet/torrent content. Implements the same interface as SoulseekClient, QobuzClient, TidalDownloadClient @@ -277,11 +280,12 @@ class LidarrDownloadClient: root_folder = self._get_root_folder() quality_profile_id = self._get_quality_profile_id() + metadata_profile_id = self._get_metadata_profile_id() add_artist = { 'foreignArtistId': artist_data.get('foreignArtistId', ''), 'artistName': artist_data.get('artistName', ''), 'qualityProfileId': quality_profile_id, - 'metadataProfileId': 1, + 'metadataProfileId': metadata_profile_id, 'rootFolderPath': root_folder, 'monitored': False, 'addOptions': {'monitor': 'none', 'searchForMissingAlbums': False}, @@ -331,8 +335,15 @@ class LidarrDownloadClient: self._set_error(download_id, f'Failed to trigger download: {e}') return - # Step 4: Poll queue for progress + # Step 4: Poll until Lidarr reports the album has imported files. + # + # Old approach used `for/else` with `break` from the inner queue + # loop, but inner-break only escaped the queue iteration — the + # outer poll loop kept spinning even after we'd detected + # completion. Replaced with an explicit `download_complete` flag + # that breaks the OUTER loop once trackFileCount > 0. max_polls = 600 # 10 minutes max + download_complete = False for poll in range(max_polls): if self.shutdown_check and self.shutdown_check(): self._set_error(download_id, 'Server shutting down') @@ -350,72 +361,125 @@ class LidarrDownloadClient: for item in queue['records']: item_album = item.get('album', {}) if item_album.get('foreignAlbumId') == album.get('foreignAlbumId', ''): - # Found our download in the queue + # Surface progress while still downloading. status = item.get('status', '').lower() - progress = 100.0 - (item.get('sizeleft', 0) / max(item.get('size', 1), 1) * 100) + size_left = item.get('sizeleft', 0) + size_total = max(item.get('size', 1), 1) + progress = 100.0 - (size_left / size_total * 100) with self._download_lock: self.active_downloads[download_id]['progress'] = min(progress, 95.0) - if status in ('completed', 'imported'): - break - elif status in ('failed', 'warning'): + if status in ('failed', 'warning'): self._set_error(download_id, f'Lidarr download failed: {status}') return + # 'completed' / 'imported' in the queue is + # transient — Lidarr drops the item once + # import finishes. Don't break here; let the + # trackFileCount check below decide. - else: - # Not in queue — might be completed already - # Check if album has files - if poll > 10: # Give it at least 10 seconds - album_check = self._api_get(f'album/{lidarr_album_id}') - if album_check and album_check.get('statistics', {}).get('trackFileCount', 0) > 0: - break - else: - # No queue data — check if completed - if poll > 10: - album_check = self._api_get(f'album/{lidarr_album_id}') - if album_check and album_check.get('statistics', {}).get('trackFileCount', 0) > 0: - break + # Authoritative completion signal: album has imported + # files. Cheap to call (single GET on a known id) and + # works even when the queue record disappeared between + # polls. + if poll > 5: # Give Lidarr a few seconds to start + album_check = self._api_get(f'album/{lidarr_album_id}') + if (album_check + and album_check.get('statistics', {}).get('trackFileCount', 0) > 0): + download_complete = True + break except Exception as e: logger.debug(f"Queue poll error: {e}") time.sleep(1) - else: + + if not download_complete: self._set_error(download_id, 'Download timed out') return - # Step 5: Find and import downloaded files + # Step 5: Find and import the wanted track. + # + # Lidarr grabs whole albums; SoulSync's matched-context + # post-processing wants the SPECIFIC track the user + # requested. Old behavior copied every track in the album + # and reported `imported_files[0]` as `file_path` — which + # almost always pointed to track 1, not the user's actual + # track. Post-processing then tagged track 1 with the + # requested track's metadata. Misfiling guaranteed. + # + # New behavior: identify the wanted track by title (parsed + # from display_name), look up its trackFile via Lidarr's + # `track` API, copy ONLY that file. For album-level + # dispatches (no specific track in display_name), fall back + # to copying the first imported file so existing + # album-grab UX still works. with self._download_lock: self.active_downloads[download_id]['progress'] = 96.0 try: - # Get track files from Lidarr - track_files = self._api_get('trackfile', params={'albumId': lidarr_album_id}) - if not track_files: - self._set_error(download_id, 'No files found after download') - return + wanted_title = self._extract_wanted_track_title(display_name) + wanted_src = self._pick_track_file_for_wanted(lidarr_album_id, wanted_title) - # Copy files to SoulSync's download path - imported_files = [] - for tf in track_files: - src_path = tf.get('path', '') - if src_path and os.path.exists(src_path): - dst_path = os.path.join(str(self.download_path), os.path.basename(src_path)) - try: - shutil.copy2(src_path, dst_path) - imported_files.append(dst_path) - except Exception as e: - logger.warning(f"Failed to copy {src_path}: {e}") + if wanted_src: + # Copy ONLY the matched track. Other album files stay + # in Lidarr's root folder and will be cleaned up by + # the cleanup step (Step 6) when configured. + dst_path = os.path.join(str(self.download_path), + os.path.basename(wanted_src)) + try: + shutil.copy2(wanted_src, dst_path) + except Exception as e: + self._set_error(download_id, f'Failed to copy wanted track: {e}') + return - if imported_files: with self._download_lock: self.active_downloads[download_id]['state'] = 'Completed, Succeeded' self.active_downloads[download_id]['progress'] = 100.0 - self.active_downloads[download_id]['file_path'] = imported_files[0] - logger.info(f"Lidarr download complete: {display_name} ({len(imported_files)} files)") + self.active_downloads[download_id]['file_path'] = dst_path + logger.info( + f"Lidarr download complete: {display_name} " + f"-> {os.path.basename(dst_path)}" + ) else: - self._set_error(download_id, 'Failed to import files') + # No specific track wanted (album dispatch) OR fuzzy + # match failed. Fall back to copying the first imported + # file so something always lands on disk; album-level + # callers still get a usable file_path. + track_files = self._api_get('trackfile', params={'albumId': lidarr_album_id}) + if not track_files: + self._set_error(download_id, 'No files found after download') + return + + imported_files = [] + for tf in track_files: + src_path = tf.get('path', '') + if src_path and os.path.exists(src_path): + dst_path = os.path.join(str(self.download_path), + os.path.basename(src_path)) + try: + shutil.copy2(src_path, dst_path) + imported_files.append(dst_path) + except Exception as e: + logger.warning(f"Failed to copy {src_path}: {e}") + + if imported_files: + with self._download_lock: + self.active_downloads[download_id]['state'] = 'Completed, Succeeded' + self.active_downloads[download_id]['progress'] = 100.0 + self.active_downloads[download_id]['file_path'] = imported_files[0] + if wanted_title: + logger.warning( + f"Lidarr: wanted track '{wanted_title}' not matched in album " + f"— falling back to first imported file ({len(imported_files)} total)" + ) + else: + logger.info( + f"Lidarr album-level download complete: {display_name} " + f"({len(imported_files)} files)" + ) + else: + self._set_error(download_id, 'Failed to import files') except Exception as e: self._set_error(download_id, f'Import failed: {e}') @@ -426,8 +490,8 @@ class LidarrDownloadClient: try: self._api_delete(f'album/{lidarr_album_id}', params={'deleteFiles': 'false'}) logger.debug(f"Cleaned up album {lidarr_album_id} from Lidarr") - except Exception: - pass + except Exception as e: + logger.debug("Lidarr album cleanup failed: %s", e) except Exception as e: logger.error(f"Lidarr download thread failed: {e}") @@ -440,6 +504,102 @@ class LidarrDownloadClient: self.active_downloads[download_id]['error'] = error logger.error(f"Lidarr download error: {error}") + @staticmethod + def _extract_wanted_track_title(display_name: str) -> str: + """Pull the track title out of the dispatch display string. + + ``_search_sync`` builds two display shapes: + - Track dispatch: ``f"{artist} - {album} - {track_title}"`` + - Album dispatch: ``f"{artist} - {album}"`` + + Need >=3 parts to confidently identify a track. 2-part strings + are album-level dispatches — return empty so the caller falls + back to copying the first file (correct behavior for "give me + the whole album"). Track titles that themselves contain ``' - '`` + (e.g. live versions) get rejoined from parts[2:]. + """ + if not display_name: + return '' + parts = display_name.split(' - ') + if len(parts) < 3: + return '' + return ' - '.join(parts[2:]).strip() + + def _pick_track_file_for_wanted(self, lidarr_album_id: int, + wanted_title: str) -> Optional[str]: + """Find the on-disk path of the imported file matching the wanted track. + + Walks Lidarr's `track` API to map track titles → trackFileIds, + then resolves the trackFileId to a path via `trackfile`. Returns + None when the album has no usable wanted-track match (caller + falls back to the first imported file in that case so + album-level dispatches still work). + """ + if not wanted_title: + return None + + tracks = self._api_get('track', params={'albumId': lidarr_album_id}) + if not tracks or not isinstance(tracks, list): + return None + + # Normalize for case-insensitive fuzzy match. Lidarr's track titles + # come from MusicBrainz so they're usually canonical, but + # punctuation / casing varies. + wanted_norm = self._normalize_for_match(wanted_title) + best_track_file_id: Optional[int] = None + best_score = 0.0 + for t in tracks: + track_title = t.get('title', '') or '' + track_file_id = t.get('trackFileId') + if not track_file_id: + continue + score = self._title_similarity(wanted_norm, + self._normalize_for_match(track_title)) + if score > best_score: + best_score = score + best_track_file_id = track_file_id + + # 0.7 threshold avoids picking the wrong track when none match + # well — caller falls back to first-imported behavior in that case. + if best_score < 0.7 or best_track_file_id is None: + return None + + # Resolve trackFileId → path. /trackfile/{id} returns one record. + tf = self._api_get(f'trackfile/{best_track_file_id}') + if not tf: + return None + path = tf.get('path', '') + if path and os.path.exists(path): + return path + return None + + @staticmethod + def _normalize_for_match(s: str) -> str: + """Lower + strip punctuation + collapse whitespace for fuzzy compare.""" + if not s: + return '' + cleaned = re.sub(r'[^\w\s]', '', s.lower()) + return ' '.join(cleaned.split()) + + @staticmethod + def _title_similarity(a: str, b: str) -> float: + """Cheap title similarity: equal → 1.0, substring → 0.85, + token overlap ratio otherwise. Avoids pulling SequenceMatcher + for every comparison since this runs in the hot download path.""" + if not a or not b: + return 0.0 + if a == b: + return 1.0 + if a in b or b in a: + return 0.85 + a_tokens = set(a.split()) + b_tokens = set(b.split()) + if not a_tokens or not b_tokens: + return 0.0 + intersection = a_tokens & b_tokens + union = a_tokens | b_tokens + return len(intersection) / len(union) if union else 0.0 + async def get_all_downloads(self) -> List[DownloadStatus]: with self._download_lock: return [self._to_status(dl) for dl in self.active_downloads.values()] @@ -536,3 +696,22 @@ class LidarrDownloadClient: if p.get('name', '').lower() == self._quality_profile.lower(): return p['id'] return profiles[0].get('id', 1) if profiles else 1 + + def _get_metadata_profile_id(self) -> int: + """Resolve a usable metadataProfileId for adding artists. + + Lidarr requires `metadataProfileId` when creating artist records. + The default profile is usually id=1, but on installs where the + user deleted/recreated profiles, that id may not exist — leading + to the API rejecting the artist-add with a 400. Fetch live to + pick whatever's actually configured. Falls back to 1 only when + the API call fails entirely (preserves previous behavior so this + change can't make things worse). + """ + profiles = self._api_get('metadataprofile') + if profiles and isinstance(profiles, list): + for p in profiles: + pid = p.get('id') + if isinstance(pid, int): + return pid + return 1 diff --git a/core/listenbrainz_manager.py b/core/listenbrainz_manager.py index ad964646..914a85ce 100644 --- a/core/listenbrainz_manager.py +++ b/core/listenbrainz_manager.py @@ -302,8 +302,8 @@ class ListenBrainzManager: # Fallback to first image if images: return images[0].get('thumbnails', {}).get('small') or images[0].get('image') - except: - pass + except Exception as e: + logger.debug("cover-art fetch: %s", e) return None diff --git a/core/listening_stats_worker.py b/core/listening_stats_worker.py index 72bdaf16..e2445aac 100644 --- a/core/listening_stats_worker.py +++ b/core/listening_stats_worker.py @@ -19,13 +19,17 @@ logger = get_logger("listening_stats_worker") class ListeningStatsWorker: """Background worker that polls media servers for play data.""" - def __init__(self, database, config_manager, plex_client=None, - jellyfin_client=None, navidrome_client=None): + def __init__(self, database, config_manager, media_server_engine=None): + """Initialize the worker. + + ``media_server_engine`` owns the per-server clients (Plex / + Jellyfin / Navidrome). The worker resolves the active server's + client through ``self._engine.client(name)`` instead of holding + per-server kwargs. + """ self.db = database self.config_manager = config_manager - self.plex_client = plex_client - self.jellyfin_client = jellyfin_client - self.navidrome_client = navidrome_client + self._engine = media_server_engine # Worker state self.running = False @@ -145,13 +149,11 @@ class ListeningStatsWorker: logger.info(f"Polling {active_server} for listening data...") self.current_item = f"Polling {active_server}..." - client = None - if active_server == 'plex' and self.plex_client: - client = self.plex_client - elif active_server == 'jellyfin' and self.jellyfin_client: - client = self.jellyfin_client - elif active_server == 'navidrome' and self.navidrome_client: - client = self.navidrome_client + client = self._engine.client(active_server) if self._engine else None + # SoulSync standalone has no listening data; only the three + # streaming servers contribute. Mirror the legacy guard here. + if active_server not in ('plex', 'jellyfin', 'navidrome'): + client = None if not client: logger.warning(f"No client available for active server: {active_server}") @@ -505,7 +507,7 @@ class ListeningStatsWorker: if conn: try: conn.close() - except Exception: + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down pass def _resolve_db_track_id(self, title, artist): diff --git a/core/matching_engine.py b/core/matching_engine.py index da1fb9bc..9abb2d06 100644 --- a/core/matching_engine.py +++ b/core/matching_engine.py @@ -7,8 +7,11 @@ from utils.logging_config import get_logger from config.settings import config_manager from core.spotify_client import Track as SpotifyTrack -from core.plex_client import PlexTrackInfo -from core.soulseek_client import TrackResult, AlbumResult +from core.media_server.types import TrackInfo +# TrackResult / AlbumResult moved out of core.soulseek_client into the +# neutral download_plugins package (download PR's Gap 1 lift). Import +# from the new location. +from core.download_plugins.types import TrackResult, AlbumResult logger = get_logger("matching_engine") @@ -16,7 +19,7 @@ logger = get_logger("matching_engine") @dataclass class MatchResult: spotify_track: SpotifyTrack - plex_track: Optional[PlexTrackInfo] + plex_track: Optional[TrackInfo] confidence: float match_type: str @@ -302,7 +305,7 @@ class MusicMatchingEngine: confidence = (title_score * 0.60) + (artist_score * 0.30) + (duration_score * 0.10) return confidence, "standard_match" - def calculate_match_confidence(self, spotify_track: SpotifyTrack, plex_track: PlexTrackInfo) -> Tuple[float, str]: + def calculate_match_confidence(self, spotify_track: SpotifyTrack, plex_track: TrackInfo) -> Tuple[float, str]: """Calculates a confidence score using a prioritized model, starting with a strict 'core' title check.""" return self.score_track_match( source_title=spotify_track.name, @@ -313,7 +316,7 @@ class MusicMatchingEngine: candidate_duration_ms=plex_track.duration if plex_track.duration else 0 ) - def find_best_match(self, spotify_track: SpotifyTrack, plex_tracks: List[PlexTrackInfo]) -> MatchResult: + def find_best_match(self, spotify_track: SpotifyTrack, plex_tracks: List[TrackInfo]) -> MatchResult: """Finds the best Plex track match from a list of candidates.""" best_match = None best_confidence = 0.0 diff --git a/core/media_server/__init__.py b/core/media_server/__init__.py new file mode 100644 index 00000000..0dc6269c --- /dev/null +++ b/core/media_server/__init__.py @@ -0,0 +1,39 @@ +"""Media server engine — central registry-backed access to the +per-server clients (Plex, Jellyfin, Navidrome, SoulSync standalone). + +Companion to the download engine refactor — same architectural +shape applied to the read-side of the library. Pre-refactor +web_server.py held four separate per-server globals +(``plex_client`` / ``jellyfin_client`` / ``navidrome_client`` / +``soulsync_library_client``) that every dispatch site reached +individually. This package replaces those globals with a single +engine that owns the client instances + a generic +``engine.client(name)`` accessor. + +The 18-or-so ``if active_server == 'plex' / 'jellyfin' / ...`` +chains in web_server.py that do server-specific work (Plex raw +playlist API vs Jellyfin / Navidrome client methods returning +different shapes) stay explicit at the call site per the "lift +what's truly shared" standard — but they reach the per-server +client through ``engine.client(name)`` rather than the legacy +globals. The four uniform-shape ``is_connected`` chains were the +only ones genuinely shared and are now ``engine.is_connected()``. + +See ``docs/media-server-engine-refactor-plan.md`` for the full +phased plan. + +Note: only ``MediaServerClient`` is re-exported here. The engine + +registry are NOT — importing the registry triggers eager imports +of every per-server client class, and those clients now inherit +``MediaServerClient`` (Cin-1), so re-exporting them here would +form a circular import the moment a client tried to resolve its +base class. Import them directly from their submodules: + from core.media_server.engine import MediaServerEngine + from core.media_server.registry import build_default_registry +""" + +from core.media_server.contract import MediaServerClient + +__all__ = [ + "MediaServerClient", +] diff --git a/core/media_server/contract.py b/core/media_server/contract.py new file mode 100644 index 00000000..d051482a --- /dev/null +++ b/core/media_server/contract.py @@ -0,0 +1,110 @@ +"""Canonical contract for media server clients. + +Narrow on purpose. Protocol body declares ONLY the methods every +registered client actually implements today — keeps the static +contract honest. Server-specific extras (Plex's +``set_music_library_by_name``, Jellyfin's user picker, Navidrome's +music folder filter, SoulSync's filesystem rescan) and methods that +most-but-not-all servers implement (``search_tracks`` on Plex / +Navidrome but not Jellyfin; ``get_recently_added_albums`` on +Jellyfin / Navidrome / SoulSync but not Plex) stay off the Protocol +and are reached through ``engine.client(name)`` directly. + +The contract is a Protocol (structural typing) rather than an ABC — +existing PlexClient / JellyfinClient / NavidromeClient / +SoulSyncClient grew the same shape independently because every +caller needed the same four calls. This file just makes that +implicit contract explicit + the conformance test pins it. +""" + +from __future__ import annotations + +from typing import Any, List, Protocol, runtime_checkable + + +@runtime_checkable +class MediaServerClient(Protocol): + """Structural contract every media server client must satisfy. + + ``runtime_checkable`` lets ``isinstance(client, MediaServerClient)`` + work, but it ONLY checks method names — not signatures. The + conformance test in ``tests/media_server/test_conformance.py`` + does the deeper class-level check via REQUIRED_METHODS. + """ + + # ------------------------------------------------------------------ + # Connection / lifecycle — required, every server implements + # ------------------------------------------------------------------ + + def is_connected(self) -> bool: + """Cheap probe — does the client have a live connection / + token / session right now? Used by the dashboard status + indicators + endpoint guards.""" + ... + + def ensure_connection(self) -> bool: + """Re-auth or reconnect if needed. May make a network call. + Returns True if connection is usable after the call.""" + ... + + # ------------------------------------------------------------------ + # Library reads — required, every server implements + # ------------------------------------------------------------------ + + def get_all_artists(self) -> List[Any]: + """Return every artist the server knows about. Each item is + a server-specific wrapper object (PlexArtist, JellyfinArtist, + NavidromeArtist, SoulSyncArtist) — caller treats them + opaquely.""" + ... + + def get_all_album_ids(self) -> set: + """Return the set of every album ID in the library. ID + format is server-native — caller doesn't introspect.""" + ... + + +# --------------------------------------------------------------------------- +# Required method set — pinned by the conformance test. Mirrors the +# Protocol body exactly so static + runtime contracts can't drift. +# --------------------------------------------------------------------------- + +REQUIRED_METHODS = { + 'is_connected', + 'ensure_connection', + 'get_all_artists', + 'get_all_album_ids', +} + +# Methods that exist on SOME servers but NOT all, listed here for +# discoverability. The conformance test does NOT enforce these. Callers +# that need one reach the per-server client directly via +# ``engine.client(name).`` rather than going through the engine, +# since the engine has no uniform safe-default that fits every method. +# +# Coverage today (audited 2026-05): +# search_tracks: Plex ✓, Navidrome ✓, Jellyfin ✗, SoulSync ✗ +# get_recently_added_albums: Jellyfin ✓, Navidrome ✓, SoulSync ✓, Plex ✗ (uses recentlyAdded() on music library) +# trigger_library_scan / is_library_scanning: Plex ✓, Jellyfin ✓, Navidrome ✓, SoulSync ✗ (filesystem walks in-process) +# get_library_stats: Plex ✓, Jellyfin ✓, Navidrome ✓, SoulSync ✗ +# create_playlist / update_playlist / get_all_playlists / etc: Plex ✓, Jellyfin ✓, Navidrome ✓, SoulSync ✗ +# update_artist_*, update_album_poster, update_track_metadata: Plex ✓, Jellyfin partial, Navidrome stubs, SoulSync ✗ +KNOWN_PER_SERVER_METHODS = ( + 'search_tracks', + 'get_recently_added_albums', + 'trigger_library_scan', + 'is_library_scanning', + 'get_library_stats', + 'create_playlist', + 'update_playlist', + 'copy_playlist', + 'get_all_playlists', + 'get_playlist_by_name', + 'get_play_history', + 'get_track_play_counts', + 'update_artist_genres', + 'update_artist_poster', + 'update_album_poster', + 'update_artist_biography', + 'update_track_metadata', +) diff --git a/core/media_server/engine.py b/core/media_server/engine.py new file mode 100644 index 00000000..d65a52ba --- /dev/null +++ b/core/media_server/engine.py @@ -0,0 +1,209 @@ +"""MediaServerEngine — central registry-backed access to media server clients. + +Honest scope: the engine OWNS the per-server client instances and +exposes a small set of generic accessors so callers don't need +per-server attribute reaches. Most actual cross-server dispatch in +web_server.py (playlist add / remove / replace, per-server metadata +sync, deep scan with server-specific cache strategies) is genuinely +different per server and stays explicit in the call site — the +engine just provides the canonical client lookup so those sites +reach via ``engine.client(name)`` instead of separate globals. + +Surface: +- ``client(name)`` / ``active_client()`` — name → client lookup +- ``active_server`` — config-driven active server name +- ``is_connected()`` — only cross-server dispatch with real callers + today (dashboard status indicators); kept as the canonical example +- ``configured_clients()`` — replaces the legacy per-server + ``if X and X.is_connected()`` chains in web_server.py +- ``reload_config(name=None)`` — generic dispatch instead of + per-client reload calls + +Per-method engine wrappers for ``get_all_artists`` / ``search_tracks`` +/ ``trigger_library_scan`` / etc. were on an earlier draft but had no +production callers — every consumer reaches the active client directly +through ``sync_service._get_active_media_client()`` or +``engine.client(name)`` and calls the per-server method itself. Cut +per the "no premature abstraction" standard. + +Engine is constructed once during web_server.py init and held as a +process-wide singleton via ``set_media_server_engine`` / +``get_media_server_engine``, mirroring the metadata + download +engine factory shape. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from utils.logging_config import get_logger + +from core.media_server.contract import MediaServerClient +from core.media_server.registry import MediaServerRegistry, build_default_registry + +logger = get_logger("media_server.engine") + + +class MediaServerEngine: + """Registry-backed access to the per-server media clients. + + Owns the per-server client instances + a small set of generic + accessors (``client(name)`` / ``active_client()`` / + ``configured_clients()`` / ``reload_config(name)``) so call sites + don't reach for separate per-server globals. The one cross-server + dispatch wrapper kept on the engine — ``is_connected()`` — + backs the dashboard status indicators that have multiple call + sites; everything else dispatches per-server in the call site + itself, reaching the relevant client through ``engine.client(name)``. + """ + + def __init__( + self, + registry: Optional[MediaServerRegistry] = None, + active_server_resolver=None, + clients: Optional[Dict[str, Any]] = None, + ) -> None: + """Initialize the engine. + + Args: + registry: Plugin registry. Defaults to the four built-in + servers (Plex, Jellyfin, Navidrome, SoulSync). + active_server_resolver: Callable returning the current + active server name (e.g. ``'plex'``). Defaults to + ``config_manager.get_active_media_server``. Tests + inject a custom resolver to switch active server + without touching real config. + clients: Pre-built {name: client_instance} dict. When + provided, the engine wraps these instances directly + instead of asking the registry to construct fresh + ones. web_server.py uses this so the engine + shares the same client objects as the + pre-existing global variables (no double-init). + """ + self.registry = registry if registry is not None else build_default_registry() + + if clients is not None: + # Wrap pre-built instances (production case from web_server.py + # init). Skip registry.initialize() — we already have the + # instances, hand them off via the registry's public + # set_instance(name, client) method so internal storage stays + # encapsulated. + for name, client in clients.items(): + self.registry.set_instance(name, client) + # Mark any registered-but-not-supplied as failed init so + # active_client() returns None for them. + for name in self.registry.names(): + if self.registry.get(name) is None and name not in clients: + self.registry.set_instance(name, None) + else: + self.registry.initialize() + + if active_server_resolver is None: + from config.settings import config_manager + active_server_resolver = config_manager.get_active_media_server + self._resolve_active = active_server_resolver + + # ------------------------------------------------------------------ + # Client lookup — generic accessors that replace per-server + # attribute reaches in callers. + # ------------------------------------------------------------------ + + def client(self, name: str) -> Optional[MediaServerClient]: + """Return the client instance for the given server name, or + None if it's not registered / failed to initialize. Used by + callers that need a server-specific method beyond the + contract surface.""" + return self.registry.get(name) + + @property + def active_server(self) -> str: + """The currently-selected media server name.""" + return self._resolve_active() + + def active_client(self) -> Optional[MediaServerClient]: + """The client for the currently-active server.""" + return self.registry.get(self.active_server) + + def is_connected(self) -> bool: + """Active server's connection state. False if no active + client (registered but failed to initialize). The dashboard + status indicators + endpoint guards rely on this — the only + cross-server dispatch wrapper kept on the engine because it + actually has callers.""" + client = self.active_client() + if client is None: + return False + try: + return client.is_connected() + except Exception as exc: + logger.warning("%s is_connected raised: %s", self.active_server, exc) + return False + + def configured_clients(self) -> Dict[str, MediaServerClient]: + """Return ``{name: client}`` for every server that's both + registered AND reports ``is_connected() == True``. Replaces + the legacy per-server `if X and X.is_connected(): ...` + chains in web_server.py. + + ``is_connected`` is in REQUIRED_METHODS, so every client the + registry yields here implements it — no hasattr guard needed. + """ + result: Dict[str, MediaServerClient] = {} + for name, client in self.registry.all_clients(): + try: + if client.is_connected(): + result[name] = client + except Exception as exc: + logger.warning("%s is_connected raised in configured_clients: %s", name, exc) + return result + + def reload_config(self, name: Optional[str] = None) -> bool: + """Reload config on a single server (or every server when + ``name`` is None). Generic dispatch — caller passes the name + instead of reaching for ``plex_client.reload_config()`` + / ``jellyfin_client.reload_config()`` directly. Servers + without a ``reload_config`` method are silently skipped. + """ + names = [name] if name else list(self.registry.names()) + ok = True + for n in names: + client = self.client(n) + if client is None or not hasattr(client, 'reload_config'): + continue + try: + client.reload_config() + except Exception as exc: + logger.warning("%s reload_config failed: %s", n, exc) + ok = False + return ok + + +# --------------------------------------------------------------------------- +# Singleton accessor — mirrors the get_metadata_engine() / +# get_download_orchestrator() pattern so callers that don't need a +# custom registry use this instead of instantiating MediaServerEngine +# directly. web_server.py constructs the singleton at startup and +# installs it via ``set_media_server_engine`` so the factory + the +# global handle share state. +# --------------------------------------------------------------------------- + +_default_engine: Optional['MediaServerEngine'] = None + + +def get_media_server_engine() -> 'MediaServerEngine': + """Return (lazily creating) the process-wide MediaServerEngine + singleton. Mirrors the ``get_metadata_engine()`` / + ``get_download_orchestrator()`` shape.""" + global _default_engine + if _default_engine is None: + _default_engine = MediaServerEngine() + return _default_engine + + +def set_media_server_engine(engine: Optional['MediaServerEngine']) -> None: + """Set the process-wide singleton. Used by web_server.py at boot + to install the engine it constructs (with the pre-built per-client + instances) as the default for callers reaching via + ``get_media_server_engine()``.""" + global _default_engine + _default_engine = engine diff --git a/core/media_server/registry.py b/core/media_server/registry.py new file mode 100644 index 00000000..03fe20b3 --- /dev/null +++ b/core/media_server/registry.py @@ -0,0 +1,142 @@ +"""Media server plugin registry. + +Single source of truth for which servers exist, what their canonical +names are, and which client class implements each. Replaces the +historic web_server.py pattern of holding 4 separate client globals +that every dispatch site reached individually. + +Server-specific dispatch chains in web_server.py (playlist add / +remove / replace, per-server metadata sync, etc.) still hand-branch +on ``active_server == X`` because the work each server does at those +sites is genuinely different — but they reach the per-server CLIENT +through ``engine.client(name)`` (which goes through this registry) +instead of separate globals. + +Adding a new server (Subsonic / Emby) = one ``register`` call here + +the new client class. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Dict, Iterator, List, Optional, Tuple + +from utils.logging_config import get_logger + +from core.media_server.contract import MediaServerClient + +# Eager imports for the same import-order reason the download plugin +# registry uses them (some integration tests inject mock modules into +# sys.modules at collection time; lazy import would bind to the mock). +from core.jellyfin_client import JellyfinClient +from core.navidrome_client import NavidromeClient +from core.plex_client import PlexClient +from core.soulsync_client import SoulSyncClient + +logger = get_logger("media_server.registry") + + +@dataclass(frozen=True) +class ServerSpec: + """Static descriptor for a media server. ``factory`` is the + zero-arg callable that builds the client (each server has its + own setup chain — Plex pulls token from config, Jellyfin reads + user_id, etc.).""" + + name: str + factory: Callable[[], MediaServerClient] + display_name: str + aliases: Tuple[str, ...] = field(default_factory=tuple) + + +class MediaServerRegistry: + """Holds the live client instances + name → instance lookup. + + Two-phase construction (mirrors the download plugin registry): + 1. Specs registered cheaply (just stores callable refs). + 2. ``initialize()`` calls each factory once. Failures captured + in ``init_failures`` so one broken server doesn't take down + the orchestrator. + """ + + def __init__(self) -> None: + self._specs: Dict[str, ServerSpec] = {} + self._instances: Dict[str, Optional[MediaServerClient]] = {} + self._init_failures: List[str] = [] + + def register(self, spec: ServerSpec) -> None: + if spec.name in self._specs: + raise ValueError(f"Server already registered: {spec.name}") + self._specs[spec.name] = spec + + def initialize(self) -> None: + for spec in self._specs.values(): + try: + instance = spec.factory() + self._instances[spec.name] = instance + except Exception as exc: + logger.error("%s media server client failed to initialize: %s", spec.display_name, exc) + self._init_failures.append(spec.display_name) + self._instances[spec.name] = None + + def set_instance(self, name: str, instance: Optional[MediaServerClient]) -> None: + """Stash a pre-built client instance into the registry without + running the spec's factory. Used by callers (web_server.py at + boot) that already constructed the per-server clients and want + the engine to wrap those exact instances rather than build new + ones. Replaces the old pattern of reaching into ``_instances`` + directly from the engine.""" + self._instances[name] = instance + + @property + def init_failures(self) -> List[str]: + return list(self._init_failures) + + def get(self, name: str) -> Optional[MediaServerClient]: + if not name: + return None + if name in self._instances: + return self._instances[name] + for spec in self._specs.values(): + if name in spec.aliases: + return self._instances.get(spec.name) + return None + + def get_spec(self, name: str) -> Optional[ServerSpec]: + if name in self._specs: + return self._specs[name] + for spec in self._specs.values(): + if name in spec.aliases: + return spec + return None + + def display_name(self, name: str) -> str: + spec = self.get_spec(name) + return spec.display_name if spec else name + + def names(self) -> List[str]: + return list(self._specs.keys()) + + def all_clients(self) -> Iterator[Tuple[str, MediaServerClient]]: + """Yield (name, client) for every successfully-initialized + server. Used by cross-server operations.""" + for name, instance in self._instances.items(): + if instance is not None: + yield name, instance + + +def build_default_registry() -> MediaServerRegistry: + """Construct the registry with SoulSync's four built-in media + servers. Called once during MediaServerEngine construction. + + Adding a server (e.g. Subsonic, Emby) = one ``register`` call + here + the new client class. No dispatch-site changes required. + """ + registry = MediaServerRegistry() + + registry.register(ServerSpec(name='plex', factory=PlexClient, display_name='Plex')) + registry.register(ServerSpec(name='jellyfin', factory=JellyfinClient, display_name='Jellyfin')) + registry.register(ServerSpec(name='navidrome', factory=NavidromeClient, display_name='Navidrome')) + registry.register(ServerSpec(name='soulsync', factory=SoulSyncClient, display_name='SoulSync Library')) + + return registry diff --git a/core/media_server/types.py b/core/media_server/types.py new file mode 100644 index 00000000..30fdbe09 --- /dev/null +++ b/core/media_server/types.py @@ -0,0 +1,132 @@ +"""Shared dataclasses for the media-server contract. + +Plex / Jellyfin / Navidrome clients all surfaced near-identical +``XTrackInfo`` and ``XPlaylistInfo`` shapes (id, title, artist, +album, duration, track_number, year, optional rating) because every +consumer needed the same shape downstream. The per-server class +names were a copy-paste artifact, not a real contract difference. + +This module owns the canonical types. Plex's existing classmethod +constructors (``TrackInfo.from_plex_track``, +``PlaylistInfo.from_plex_playlist``) live here. Jellyfin and +Navidrome currently construct ``TrackInfo`` inline at their call +sites — lifting those into matching ``from_jellyfin_dict`` / +``from_navidrome_dict`` classmethods is a clean followup but isn't +needed for the dataclass unification this module ships. + +Heavy server SDK types (``PlexTrack``) imported under TYPE_CHECKING +so this module stays import-light. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, List, Optional + +if TYPE_CHECKING: + # plexapi types — only loaded when type-checking; runtime + # paths through from_plex_track accept whatever PlexClient + # passes through. + from plexapi.audio import Track as PlexTrack + from plexapi.playlist import Playlist as PlexPlaylist + + +@dataclass +class TrackInfo: + """Canonical track-shape returned by media server clients. + + Plex, Jellyfin, and Navidrome each defined their own near-identical + ``XTrackInfo`` dataclass (SoulSync standalone uses richer per-track + wrappers and doesn't surface this exact shape). Lifted to one + canonical type here so consumers (matching engine, sync service, + library scanners) get a single import. + """ + + id: str + title: str + artist: str + album: str + duration: int + track_number: Optional[int] = None + year: Optional[int] = None + rating: Optional[float] = None + + # ------------------------------------------------------------------ + # Per-server constructors — mirror Cin's metadata Album.from_X_dict + # pattern. Each one knows ONE server's wire shape. + # ------------------------------------------------------------------ + + @classmethod + def from_plex_track(cls, track: 'PlexTrack') -> 'TrackInfo': + """Build a TrackInfo from a plexapi PlexTrack. + + Defensive: tracks may be missing artist or album metadata in + Plex (especially fan-uploaded content); fall back to + "Unknown Artist" / "Unknown Album" instead of raising. + """ + # Imported lazily so this module stays import-light. plexapi + # is heavy and pulls in network + ssl deps just to define + # exception types. + from plexapi.exceptions import NotFound + + try: + artist_title = track.artist().title if track.artist() else "Unknown Artist" + except (NotFound, AttributeError): + artist_title = "Unknown Artist" + + try: + album_title = track.album().title if track.album() else "Unknown Album" + except (NotFound, AttributeError): + album_title = "Unknown Album" + + return cls( + id=str(track.ratingKey), + title=track.title, + artist=artist_title, + album=album_title, + duration=track.duration, + track_number=track.trackNumber, + year=track.year, + rating=track.userRating, + ) + + +@dataclass +class PlaylistInfo: + """Canonical playlist-shape returned by every media server client. + + Same lift rationale as ``TrackInfo`` — every server defined the + same five-field dataclass + a list of tracks. + """ + + id: str + title: str + description: Optional[str] + duration: int + leaf_count: int + tracks: List[TrackInfo] = field(default_factory=list) + + # ------------------------------------------------------------------ + # Per-server constructors + # ------------------------------------------------------------------ + + @classmethod + def from_plex_playlist(cls, playlist: 'PlexPlaylist') -> 'PlaylistInfo': + """Build a PlaylistInfo from a plexapi Playlist. Skips items + that aren't audio tracks (Plex playlists can mix media types + in theory, though music libraries shouldn't).""" + from plexapi.audio import Track as PlexTrack + + tracks: List[TrackInfo] = [] + for item in playlist.items(): + if isinstance(item, PlexTrack): + tracks.append(TrackInfo.from_plex_track(item)) + + return cls( + id=str(playlist.ratingKey), + title=playlist.title, + description=playlist.summary, + duration=playlist.duration, + leaf_count=playlist.leafCount, + tracks=tracks, + ) diff --git a/core/metadata/__init__.py b/core/metadata/__init__.py index ea434e48..e7142adf 100644 --- a/core/metadata/__init__.py +++ b/core/metadata/__init__.py @@ -8,6 +8,7 @@ from core.metadata.album_tracks import ( resolve_album_reference, ) from core.metadata.artist_image import get_artist_image_url +from core.metadata.artwork import is_internal_image_host, normalize_image_url from core.metadata.cache import MetadataCache, get_metadata_cache from core.metadata.completion import ( check_album_completion, @@ -40,6 +41,13 @@ from core.metadata.registry import ( register_profile_spotify_credentials_provider, register_runtime_clients, ) +from core.metadata.status import ( + METADATA_SOURCE_STATUS_TTL, + get_metadata_source_status, + get_spotify_status, + get_status_snapshot, + invalidate_metadata_status_caches, +) from core.metadata.service import MetadataProvider, MetadataService, get_metadata_service from core.metadata.similar_artists import ( get_musicmap_similar_artists, @@ -48,6 +56,7 @@ from core.metadata.similar_artists import ( __all__ = [ "METADATA_SOURCE_PRIORITY", + "METADATA_SOURCE_STATUS_TTL", "MetadataCache", "MetadataLookupOptions", "MetadataProvider", @@ -71,6 +80,7 @@ __all__ = [ "get_hydrabase_client", "get_itunes_client", "get_metadata_cache", + "get_metadata_source_status", "get_metadata_service", "get_musicmap_similar_artists", "get_primary_client", @@ -78,11 +88,16 @@ __all__ = [ "get_spotify_client_for_profile", "get_registered_runtime_client", "get_spotify_client", + "get_spotify_status", "get_source_priority", + "get_status_snapshot", "iter_artist_discography_completion_events", "iter_musicmap_similar_artist_events", "is_hydrabase_enabled", + "is_internal_image_host", "register_profile_spotify_credentials_provider", "register_runtime_clients", + "normalize_image_url", "resolve_album_reference", + "invalidate_metadata_status_caches", ] diff --git a/core/metadata/album_mbid_cache.py b/core/metadata/album_mbid_cache.py new file mode 100644 index 00000000..ba32889f --- /dev/null +++ b/core/metadata/album_mbid_cache.py @@ -0,0 +1,175 @@ +"""Persistent MusicBrainz release-MBID cache for albums. + +The original in-memory `mb_release_cache` in `core/metadata/source.py` +maps `(normalized_album_name, artist_name) -> release_mbid` so per-track +enrichment of the same album hits the cache and writes the same +``MUSICBRAINZ_ALBUMID`` to every track's tags. That cache is a bounded +``OrderedDict`` (4096 entries) — bounded means it can evict entries +between tracks of the same album when other albums are processed in +between. Server restart drops it entirely. Either case can produce +inconsistent album MBIDs across tracks of the same album, which causes +Navidrome (and other media servers that group by album MBID) to split +the album into multiple entries. + +This module is the persistent layer behind that cache. Same key shape, +backed by a tiny SQLite table so a successful lookup remembered ONCE +applies to every future track of the same album for the lifetime of +the install — not just the bounded in-memory window. + +Strict additive design: every public function is wrapped in try/except +and degrades to a None / no-op return on any database error. The +existing in-memory cache + MusicBrainz lookup stays behind it as the +authoritative fallback. If this module breaks, downloads continue +exactly as they would today — just without the persistent benefit. +""" + +from __future__ import annotations + +import threading +from typing import Optional + +from utils.logging_config import get_logger + + +logger = get_logger("metadata.album_mbid_cache") + + +# Lazy DB accessor — the cache module shouldn't trigger MusicDatabase +# import at module-load time (circular-import risk when source.py is +# imported during database initialization). +_db_factory_lock = threading.Lock() +_db_factory = None + + +def _get_database(): + """Resolve the MusicDatabase singleton lazily. + + Returns None if anything goes wrong — callers MUST handle a None + return as "cache unavailable, fall through to MB lookup." + """ + global _db_factory + with _db_factory_lock: + if _db_factory is None: + try: + from database.music_database import get_database + _db_factory = get_database + except Exception as exc: + logger.warning(f"Persistent MBID cache: could not load database module: {exc}") + return None + try: + return _db_factory() + except Exception as exc: + logger.warning(f"Persistent MBID cache: database accessor failed: {exc}") + return None + + +def lookup(normalized_album_key: str, artist_key: str) -> Optional[str]: + """Read a cached release MBID for the given (album, artist) pair. + + Returns the stored MBID string if found, otherwise None. Never + raises — DB errors degrade silently to "cache miss" so the caller + falls through to MusicBrainz like it does today. + + Args: + normalized_album_key: Output of ``normalize_album_cache_key`` — + already lowercased and stripped of edition parentheticals. + artist_key: Lowercased artist name (caller's responsibility to + pass a normalized key — keeps the schema uniform). + """ + if not normalized_album_key or not artist_key: + return None + + db = _get_database() + if db is None: + return None + + conn = None + try: + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute( + "SELECT release_mbid FROM mb_album_release_cache " + "WHERE normalized_album_key = ? AND artist_key = ? LIMIT 1", + (normalized_album_key, artist_key), + ) + row = cursor.fetchone() + if row: + mbid = row[0] if not hasattr(row, 'keys') else row['release_mbid'] + return mbid or None + except Exception as exc: + logger.debug(f"Persistent MBID cache lookup failed: {exc}") + finally: + if conn is not None: + try: + conn.close() + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down + pass + + return None + + +def record(normalized_album_key: str, artist_key: str, release_mbid: str) -> bool: + """Persist a (album, artist) -> release_mbid mapping. + + Idempotent — uses INSERT OR REPLACE so re-recording the same key + just refreshes the timestamp. Returns True on success, False on + any failure. Failure is logged at debug level and never propagated + so a flaky DB write can't break the enrichment path. + """ + if not normalized_album_key or not artist_key or not release_mbid: + return False + + db = _get_database() + if db is None: + return False + + conn = None + try: + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute( + "INSERT OR REPLACE INTO mb_album_release_cache " + "(normalized_album_key, artist_key, release_mbid, updated_at) " + "VALUES (?, ?, ?, CURRENT_TIMESTAMP)", + (normalized_album_key, artist_key, release_mbid), + ) + conn.commit() + return True + except Exception as exc: + logger.debug(f"Persistent MBID cache record failed: {exc}") + return False + finally: + if conn is not None: + try: + conn.close() + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down + pass + + +def clear_all() -> bool: + """Wipe the persistent cache. Used by tests and by the maintenance + endpoint when a user wants to force a fresh MusicBrainz re-lookup + (e.g. after fixing widespread MBID inconsistencies).""" + db = _get_database() + if db is None: + return False + + conn = None + try: + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute("DELETE FROM mb_album_release_cache") + conn.commit() + return True + except Exception as exc: + logger.warning(f"Persistent MBID cache clear failed: {exc}") + return False + finally: + if conn is not None: + try: + conn.close() + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down + pass + + +__all__ = ["lookup", "record", "clear_all"] diff --git a/core/metadata/album_tracks.py b/core/metadata/album_tracks.py index e15a757c..37e728a4 100644 --- a/core/metadata/album_tracks.py +++ b/core/metadata/album_tracks.py @@ -2,14 +2,32 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional +from dataclasses import replace +from typing import Any, Callable, Dict, List, Optional from core.metadata import registry as metadata_registry from core.metadata.lookup import MetadataLookupOptions +from core.metadata.types import Album from utils.logging_config import get_logger logger = get_logger("metadata.album_tracks") + +# Per-source typed converter dispatch. Powers the typed path inside +# ``_build_album_info`` — when the caller knows which provider the raw +# response came from, route through the canonical Album converter +# instead of duck-typing every field. Sources missing from this map +# fall through to the legacy duck-typed path. +_TYPED_ALBUM_CONVERTERS: Dict[str, Callable[[Dict[str, Any]], Album]] = { + 'spotify': Album.from_spotify_dict, + 'itunes': Album.from_itunes_dict, + 'deezer': Album.from_deezer_dict, + 'discogs': Album.from_discogs_dict, + 'musicbrainz': Album.from_musicbrainz_dict, + 'hydrabase': Album.from_hydrabase_dict, + 'qobuz': Album.from_qobuz_dict, +} + __all__ = [ "get_album_for_source", "get_album_tracks_for_source", @@ -178,7 +196,83 @@ def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: return normalized -def _build_album_info(album_data: Any, album_id: str, album_name: str = '', artist_name: str = '') -> Dict[str, Any]: +def _build_album_info(album_data: Any, album_id: str, album_name: str = '', + artist_name: str = '', source: str = '') -> Dict[str, Any]: + """Build the canonical SoulSync internal album-info dict. + + When ``source`` is provided AND maps to a known typed converter, + routes through the canonical ``Album.from__dict()`` path — + that single converter is the source of truth for that provider's + wire shape. Falls back to the legacy duck-typed extraction when + source is empty/unknown OR when the typed converter raises (so a + converter bug can't break album resolution). + + See ``docs/metadata-types-migration.md`` for the broader plan. + """ + typed_path_succeeded = None + if source and isinstance(album_data, dict): + converter = _TYPED_ALBUM_CONVERTERS.get(source.lower()) + if converter is not None: + try: + typed_path_succeeded = _build_album_info_typed( + album_data, album_id, album_name, artist_name, converter, + ) + except Exception as exc: + logger.debug( + "Typed album_info converter failed for source %s, falling " + "back to legacy path: %s", source, exc, + ) + if typed_path_succeeded is not None: + return typed_path_succeeded + + return _build_album_info_legacy(album_data, album_id, album_name, artist_name) + + +def _build_album_info_typed(album_data: Dict[str, Any], album_id: str, + album_name: str, artist_name: str, + converter: Callable[[Dict[str, Any]], Album]) -> Dict[str, Any]: + """Typed path: convert raw → Album, apply caller fallbacks for + fields the converter couldn't fill, return canonical dict.""" + album = converter(album_data) + + # Apply caller-provided fallbacks when the converter produced + # empty values. The legacy path treated `album_id` / `album_name` + # / `artist_name` as last-resort defaults. + if not album.id: + album = replace(album, id=album_id) + if not album.name: + album = replace(album, name=album_name or album_id) + if (not album.artists or album.artists == ['Unknown Artist']) and artist_name: + album = replace(album, artists=[artist_name]) + + ctx = album.to_context_dict() + + # Preserve original `images` list shape from the raw input — the + # legacy path passed the source's full multi-resolution images + # array through verbatim. Some downstream consumers iterate the + # full list to pick a different size. + raw_images = album_data.get('images') + if isinstance(raw_images, list) and raw_images: + ctx['images'] = raw_images + # Legacy path also derived image_url from the first images entry + # when the source-specific cover field wasn't populated. Match + # that fallback so callers with Spotify-shaped raw images keep + # getting an image_url out of providers whose typed converter + # only checks source-native cover fields. + if not ctx.get('image_url'): + first = raw_images[0] + if isinstance(first, dict): + ctx['image_url'] = first.get('url') or ctx.get('image_url') + + return ctx + + +def _build_album_info_legacy(album_data: Any, album_id: str, + album_name: str, artist_name: str) -> Dict[str, Any]: + """Original duck-typed extraction. Kept as the fallback when the + typed path can't apply (unknown source, non-dict input, converter + error). Tracked for removal once every caller passes a recognized + source — see migration plan.""" images = _extract_lookup_value(album_data, 'images', default=[]) or [] if not isinstance(images, list): images = list(images) if images else [] @@ -252,7 +346,10 @@ def _build_album_tracks_payload( album_name: str = '', artist_name: str = '', ) -> Dict[str, Any]: - album_info = _build_album_info(album_data, album_id, album_name=album_name, artist_name=artist_name) + album_info = _build_album_info( + album_data, album_id, + album_name=album_name, artist_name=artist_name, source=source, + ) album_info['source'] = source album_info['_source'] = source album_info['provider'] = source diff --git a/core/metadata/artwork.py b/core/metadata/artwork.py index ca268e52..13276104 100644 --- a/core/metadata/artwork.py +++ b/core/metadata/artwork.py @@ -5,6 +5,8 @@ from __future__ import annotations import os import re import urllib.request +from ipaddress import ip_address +from urllib.parse import quote, urlparse from core.imports.context import get_import_context_album from core.metadata.common import ( @@ -17,12 +19,193 @@ from utils.logging_config import get_logger as _create_logger __all__ = [ "embed_album_art_metadata", "download_cover_art", + "is_internal_image_host", + "is_image_proxy_url", + "normalize_image_url", ] logger = _create_logger("metadata.artwork") +def normalize_image_url(thumb_url: str | None) -> str | None: + """Convert media-server image URLs into browser-safe URLs.""" + if not thumb_url: + return None + + try: + if is_image_proxy_url(thumb_url): + # Already normalized for browser use; avoid wrapping it in another proxy layer. + return thumb_url + + # Check if it's a localhost URL or relative path that needs fixing + needs_fixing = ( + thumb_url.startswith('http://localhost:') or + thumb_url.startswith('https://localhost:') or + thumb_url.startswith('http://127.0.0.1:') or + thumb_url.startswith('https://127.0.0.1:') or + thumb_url.startswith('http://host.docker.internal:') or + thumb_url.startswith('https://host.docker.internal:') or + (thumb_url.startswith('http://') and is_internal_image_host(thumb_url)) or + thumb_url.startswith('/library/') or # Plex relative paths + thumb_url.startswith('/Items/') or # Jellyfin relative paths + thumb_url.startswith('/api/') or # Old Navidrome API paths + thumb_url.startswith('/rest/') # Navidrome Subsonic API paths + ) + + if needs_fixing: + cfg = get_config_manager() + active_server = cfg.get_active_media_server() + logger.debug("Fixing URL: %s, Active server: %s", thumb_url, active_server) + + if active_server == 'plex': + plex_config = cfg.get_plex_config() + plex_base_url = plex_config.get('base_url', '') + plex_token = plex_config.get('token', '') + logger.info("Plex config - base_url: %s, token: %s...", plex_base_url, plex_token[:10]) + + if plex_base_url and plex_token: + # Extract the path from URL + if thumb_url.startswith('/library/'): + # Already a path + path = thumb_url + else: + # Full localhost URL, extract path + parsed = urlparse(thumb_url) + path = parsed.path + + # Construct proper Plex URL with token + fixed_url = f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}" + logger.info("Fixed URL: %s", fixed_url) + return _browser_safe_image_url(fixed_url) + + elif active_server == 'jellyfin': + jellyfin_config = cfg.get_jellyfin_config() + jellyfin_base_url = jellyfin_config.get('base_url', '') + jellyfin_token = jellyfin_config.get('api_key', '') + logger.info( + "Jellyfin config - base_url: %s, token: %s...", + jellyfin_base_url, + jellyfin_token[:10] if jellyfin_token else 'None', + ) + + if jellyfin_base_url: + # Extract the path from URL + if thumb_url.startswith('/Items/') or thumb_url.startswith('/api/'): + # Already a path + path = thumb_url + else: + # Full localhost URL, extract path + parsed = urlparse(thumb_url) + path = parsed.path + + # Construct proper Jellyfin URL with token + if jellyfin_token: + separator = '&' if '?' in path else '?' + fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}" + else: + fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}" + logger.info("Fixed URL: %s", fixed_url) + return _browser_safe_image_url(fixed_url) + + elif active_server == 'navidrome': + navidrome_config = cfg.get_navidrome_config() + navidrome_base_url = navidrome_config.get('base_url', '') + navidrome_username = navidrome_config.get('username', '') + navidrome_password = navidrome_config.get('password', '') + logger.info("Navidrome config - base_url: %s, username: %s", navidrome_base_url, navidrome_username) + + if navidrome_base_url and navidrome_username and navidrome_password: + # Extract the path from URL + if thumb_url.startswith('/rest/'): + # Already a Subsonic API path + path = thumb_url + else: + # Full localhost URL, extract path + parsed = urlparse(thumb_url) + path = parsed.path + + # Generate Subsonic API authentication + import hashlib + import secrets + salt = secrets.token_hex(6) + token = hashlib.md5((navidrome_password + salt).encode()).hexdigest() + + # Add authentication parameters to the URL + separator = '&' if '?' in path else '?' + auth_params = f"u={navidrome_username}&t={token}&s={salt}&v=1.16.1&c=SoulSync&f=json" + + # Construct proper Navidrome Subsonic URL + fixed_url = f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}" + logger.info("Fixed URL: %s", fixed_url) + return _browser_safe_image_url(fixed_url) + + logger.warning("No configuration found for %s or unsupported server type", active_server) + + # Return a browser-safe URL even if no server-specific rebuild was possible. + return _browser_safe_image_url(thumb_url) + + except Exception as exc: + logger.error("Error fixing image URL '%s': %s", thumb_url, exc) + return _browser_safe_image_url(thumb_url) + + +def is_image_proxy_url(url: str) -> bool: + """Return True for SoulSync image-proxy URLs, absolute or relative.""" + if not url: + return False + + try: + parsed = urlparse(url) + return parsed.path == '/api/image-proxy' + except Exception: + return False + + +def is_internal_image_host(url: str) -> bool: + """Return True when an image URL points at a host the browser likely cannot reach directly.""" + try: + parsed = urlparse(url) + host = (parsed.hostname or '').strip('[]').lower() + if not host: + return False + + if host in {'localhost', '127.0.0.1', '::1', 'host.docker.internal'}: + return True + + # Single-label hosts are usually Docker service names or local LAN aliases. + if '.' not in host: + return True + + try: + ip = ip_address(host) + return ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_reserved + except ValueError: + return False + except Exception: + return False + + +def _browser_safe_image_url(url: str) -> str: + """Return a browser-safe image URL, proxying internal hosts through SoulSync.""" + if not url: + return url + + if is_image_proxy_url(url): + return url + + if url.startswith('/api/image-proxy?url='): + return url + + if url.startswith('http://') or url.startswith('https://'): + if is_internal_image_host(url): + return f"/api/image-proxy?url={quote(url, safe='')}" + return url + + # Relative media-server paths should already have been expanded before this point. + return url + + def embed_album_art_metadata(audio_file, metadata: dict): cfg = get_config_manager() symbols = get_mutagen_symbols() @@ -137,8 +320,8 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None): from core.spotify_client import _upgrade_spotify_image_url art_url = _upgrade_spotify_image_url(art_url) - except Exception: - pass + except Exception as e: + logger.debug("upgrade spotify image url failed: %s", e) elif art_url and "mzstatic.com" in art_url: art_url = re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) if not art_url: diff --git a/core/metadata/cache.py b/core/metadata/cache.py index c52c91ac..2974c5eb 100644 --- a/core/metadata/cache.py +++ b/core/metadata/cache.py @@ -33,8 +33,8 @@ def get_metadata_cache(): try: import threading threading.Thread(target=_cache_instance.backfill_deezer_album_genres, daemon=True).start() - except Exception: - pass + except Exception as e: + logger.debug("start deezer genres backfill failed: %s", e) return _cache_instance diff --git a/core/metadata/common.py b/core/metadata/common.py index 9220cd75..ebf6ee5d 100644 --- a/core/metadata/common.py +++ b/core/metadata/common.py @@ -171,8 +171,8 @@ def get_image_dimensions(data: bytes): return w, h length = struct.unpack(">H", data[i + 2 : i + 4])[0] i += 2 + length - except Exception: - pass + except Exception as e: + logger.debug("parse JPEG dimensions failed: %s", e) return None, None diff --git a/core/metadata/discography.py b/core/metadata/discography.py index ac51e141..35f6df77 100644 --- a/core/metadata/discography.py +++ b/core/metadata/discography.py @@ -2,16 +2,53 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional from core.metadata import registry as metadata_registry from core.metadata.album_tracks import get_artist_albums_for_source from core.metadata.lookup import MetadataLookupOptions +from core.metadata.types import Album from utils.logging_config import get_logger logger = get_logger("metadata.discography") +# Per-source typed converter dispatch — same registry pattern as +# ``core/metadata/album_tracks.py`` and ``core/imports/resolution.py``. +# Discography release builders dispatch through the typed Album +# converter when the active source is known. Falls back to legacy +# duck-typed extraction below on unknown source / converter error. +_TYPED_ALBUM_CONVERTERS: Dict[str, Callable[[Dict[str, Any]], Album]] = { + 'spotify': Album.from_spotify_dict, + 'itunes': Album.from_itunes_dict, + 'deezer': Album.from_deezer_dict, + 'discogs': Album.from_discogs_dict, + 'musicbrainz': Album.from_musicbrainz_dict, + 'hydrabase': Album.from_hydrabase_dict, + 'qobuz': Album.from_qobuz_dict, +} + + +def _typed_album_for_source(release: Any, source: Optional[str]) -> Optional[Album]: + """Return a typed Album when source maps to a registered converter + and the converter succeeds. ``None`` means the caller should fall + back to the legacy duck-typed extraction. + """ + if not source or not isinstance(release, dict): + return None + converter = _TYPED_ALBUM_CONVERTERS.get(source.strip().lower()) + if converter is None: + return None + try: + return converter(release) + except Exception as exc: + logger.debug( + "Typed album converter failed for source %s in discography " + "build, falling back to legacy: %s", source, exc, + ) + return None + + def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any: if value is None: return default @@ -89,7 +126,32 @@ def _pick_best_artist_match(search_results: List[Any], artist_name: str) -> Opti return search_results[0] -def _build_discography_release_dict(release: Any, artist_id: str) -> Optional[Dict[str, Any]]: +def _build_discography_release_dict(release: Any, artist_id: str, + source: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Build a normalized discography release dict. + + When ``source`` is provided AND maps to a registered typed Album + converter, routes through ``Album.from__dict()`` and pulls + canonical fields off the typed Album. Falls back to the legacy + duck-typed extraction on unknown source / non-dict input / typed + converter error. + """ + typed_album = _typed_album_for_source(release, source) + if typed_album is not None: + if not typed_album.id: + return None + artist_name = typed_album.artists[0] if typed_album.artists else '' + return { + 'id': typed_album.id, + 'name': typed_album.name or typed_album.id, + 'artist_name': artist_name, + 'release_date': typed_album.release_date or None, + 'album_type': typed_album.album_type or 'album', + 'image_url': typed_album.image_url, + 'total_tracks': typed_album.total_tracks or 0, + 'external_urls': typed_album.external_urls or {}, + } + release_id = _extract_lookup_value(release, 'id', 'album_id', 'release_id') if not release_id: return None @@ -308,7 +370,7 @@ def get_artist_discography( seen_albums = set() for release in albums or []: - release_data = _build_discography_release_dict(release, artist_id) + release_data = _build_discography_release_dict(release, artist_id, source=active_source) if not release_data: continue @@ -341,7 +403,46 @@ def get_artist_discography( } -def _build_artist_detail_release_card(release: Dict[str, Any]) -> Optional[Dict[str, Any]]: +def _build_artist_detail_release_card(release: Dict[str, Any], + source: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Build an artist-detail release card. + + NOTE on inputs: this function may receive EITHER raw provider + release dicts (when called directly during a fresh discography + lookup) OR pre-built canonical release dicts produced by + ``_build_discography_release_dict`` (the more common case via + ``get_artist_detail_discography``). Pre-built dicts already carry + canonical keys, so the typed dispatch is a no-op for them — and + the legacy duck-typed path also handles them correctly. The typed + dispatch only kicks in when the caller passes a known source AND + the release dict matches a provider's wire shape. + """ + typed_album = _typed_album_for_source(release, source) + if typed_album is not None and typed_album.id: + release_year = None + if typed_album.release_date: + try: + release_year = str(typed_album.release_date)[:4] + except Exception: + release_year = None + + card = { + 'id': typed_album.id, + 'name': typed_album.name or typed_album.id, + 'title': typed_album.name or typed_album.id, + 'album_type': (typed_album.album_type or 'album').lower(), + 'image_url': typed_album.image_url, + 'year': release_year, + 'track_count': typed_album.total_tracks or 0, + 'owned': None, + 'track_completion': 'checking', + } + if typed_album.release_date: + card['release_date'] = typed_album.release_date + elif release_year: + card['release_date'] = f"{release_year}-01-01" + return card + release_id = _extract_lookup_value(release, 'id', 'album_id', 'release_id') if not release_id: return None diff --git a/core/metadata/enrichment.py b/core/metadata/enrichment.py index 82f462d6..6317cedf 100644 --- a/core/metadata/enrichment.py +++ b/core/metadata/enrichment.py @@ -37,6 +37,7 @@ def build_metadata_enrichment_runtime( deezer_worker: Any | None = None, audiodb_worker: Any | None = None, tidal_client: Any | None = None, + hifi_client: Any | None = None, qobuz_enrichment_worker: Any | None = None, lastfm_worker: Any | None = None, genius_worker: Any | None = None, @@ -49,6 +50,7 @@ def build_metadata_enrichment_runtime( deezer_worker=deezer_worker, audiodb_worker=audiodb_worker, tidal_client=tidal_client, + hifi_client=hifi_client, qobuz_enrichment_worker=qobuz_enrichment_worker, lastfm_worker=lastfm_worker, genius_worker=genius_worker, diff --git a/core/metadata/multi_source_search.py b/core/metadata/multi_source_search.py new file mode 100644 index 00000000..b7df0705 --- /dev/null +++ b/core/metadata/multi_source_search.py @@ -0,0 +1,241 @@ +"""Multi-source parallel metadata search. + +Both the Track Redownload modal and the Artist Enhance Quality flow +need to find the best metadata match for a known track (we have the +title + artist + duration from the user's library; we want to find +the matching entry in Spotify / iTunes / Deezer / Discogs / Hydrabase +to drive the wishlist re-download). + +Pre-extraction, redownload had a fully-fledged multi-source parallel +search (parallel ThreadPoolExecutor, per-source query optimization, +"current match" flagging via stored source IDs, per-result scoring) +while enhance had a hardcoded Spotify-direct → Spotify-search → +iTunes-fallback chain that only searched ONE source. That's why +redownload "worked" for users without Spotify (it'd find matches via +iTunes / Deezer in parallel) and enhance silently failed (single +fallback returned junk). + +This module owns the search logic. Both endpoints call +``search_all_sources`` and get back the same shape — same scoring, +same source-optimized queries, same "current match" semantics. UI +behavior diverges per-endpoint (redownload renders a picker, enhance +auto-picks the best across all sources) but the metadata-search +contract is shared. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from typing import Any, Dict, List, Optional, Tuple + +from utils.logging_config import get_logger + +logger = get_logger('metadata.multi_source_search') + + +@dataclass +class TrackQuery: + """Inputs needed to run a multi-source metadata search for one track.""" + title: str + artist: str + album: str = '' + # Library-side duration in milliseconds. Used for the duration + # similarity component of scoring; pass 0 when unknown (scoring + # falls back to a neutral 0.5 weight). + duration_ms: int = 0 + # Source-native track IDs already stored on the library track, + # used for the "is_current_match" flag in the per-result rendering + # so the UI can highlight the entry that produced the existing file. + spotify_track_id: Optional[str] = None + deezer_id: Optional[str] = None + + +@dataclass +class MultiSourceResult: + """Aggregated output from ``search_all_sources``.""" + # source_name → list of result dicts (already per-source sorted: + # is_current_match first, then descending match_score). Dict shape + # is JSON-serializable for direct return to the frontend (the + # redownload picker uses these as-is). + metadata_results: Dict[str, List[dict]] = field(default_factory=dict) + # source_name → list of source-native Track objects, parallel-indexed + # to ``metadata_results[source_name]``. Used by callers (Enhance + # Quality) that need to build a wishlist payload from the chosen + # match — the dict shape lacks per-source fields like full album + # data / external_urls / popularity that the wishlist needs. + raw_tracks: Dict[str, List[Any]] = field(default_factory=dict) + # Best match across all sources, or None if every source returned + # nothing. Shape: ``{'source': str, 'index': int, 'score': float}``. + best_match: Optional[dict] = None + + def best_track(self) -> Optional[Any]: + """Convenience: return the source-native Track object for the + cross-source best match, or None if no match was found.""" + if not self.best_match: + return None + source = self.best_match['source'] + index = self.best_match['index'] + tracks = self.raw_tracks.get(source) or [] + return tracks[index] if index < len(tracks) else None + + +def _score_match(query: TrackQuery, result: dict) -> float: + """Score one result dict against the query. + + Weights: title 0.5, artist 0.35, duration 0.15. Duration component + is neutral (0.5) when the library track has no duration on file. + + These weights match the redownload pre-extraction implementation + so existing callers keep their scoring behavior identical. + """ + title_sim = SequenceMatcher( + None, query.title.lower(), (result.get('name') or '').lower() + ).ratio() + artist_sim = SequenceMatcher( + None, query.artist.lower(), (result.get('artist') or '').lower() + ).ratio() + if query.duration_ms: + dur_diff = abs(query.duration_ms - (result.get('duration_ms') or 0)) + dur_score = max(0.0, 1.0 - dur_diff / 30000.0) + else: + dur_score = 0.5 + return round((title_sim * 0.5 + artist_sim * 0.35 + dur_score * 0.15), 3) + + +def _build_source_query(source_name: str, query: TrackQuery, clean_title: str) -> str: + """Build the source-optimized search query string. + + Deezer's API responds best to its native field-prefixed syntax + (``artist:"X" track:"Y"``) — empirically returns better matches + than a plain query for ambiguous track names. Other sources use + the artist + clean-title concatenation. + """ + if source_name == 'deezer': + return f'artist:"{query.artist}" track:"{clean_title}"' + return f"{query.artist} {clean_title}" + + +def _search_one_source(source_name: str, client: Any, + query: TrackQuery, clean_title: str + ) -> Tuple[str, List[dict], List[Any]]: + """Run one source's search with three-tier query fallback. + + Tier 1: source-optimized query (Deezer's structured form, others' plain). + Tier 2: plain ``artist + title`` if tier 1 returned nothing. + Tier 3: title-only as last resort. + + Returns ``(source_name, results, raw_tracks)``: + - ``results`` are the JSON-serializable dicts (id / name / artist / + etc.), sorted by is_current_match first, then descending match_score + - ``raw_tracks`` are the source-native Track objects, parallel-indexed + to ``results``, for callers that need richer per-source fields + than the dict surface (album_type, external_urls, etc). + """ + try: + primary_q = _build_source_query(source_name, query, clean_title) + plain_q = f"{query.artist} {clean_title}" + title_q = clean_title + + logger.info(f"[MultiSourceSearch] Searching {source_name} for: {primary_q}") + track_objs = client.search_tracks(primary_q, limit=10) + if not track_objs and primary_q != plain_q: + track_objs = client.search_tracks(plain_q, limit=10) + if not track_objs and clean_title != plain_q: + track_objs = client.search_tracks(title_q, limit=10) + logger.info(f"[MultiSourceSearch] {source_name} returned {len(track_objs)} results") + + scored: List[Tuple[dict, Any]] = [] + for t in track_objs: + r = { + 'id': str(getattr(t, 'id', '')), + 'name': getattr(t, 'name', '') or '', + 'artist': ', '.join(t.artists) if getattr(t, 'artists', None) else '', + 'album': getattr(t, 'album', '') or '', + 'duration_ms': getattr(t, 'duration_ms', 0) or 0, + 'image_url': getattr(t, 'image_url', '') or '', + 'is_current_match': False, + } + # Flag the result that backs the user's existing library + # track so the UI can highlight it. + if source_name == 'spotify' and query.spotify_track_id and r['id'] == str(query.spotify_track_id): + r['is_current_match'] = True + elif source_name == 'deezer' and query.deezer_id and r['id'] == str(query.deezer_id): + r['is_current_match'] = True + r['match_score'] = _score_match(query, r) + scored.append((r, t)) + + # Sort dict + raw track in lockstep so raw_tracks[i] is the + # source-native object behind metadata_results[source][i]. + scored.sort(key=lambda pair: (-int(pair[0]['is_current_match']), -pair[0]['match_score'])) + results = [pair[0] for pair in scored] + raw_tracks = [pair[1] for pair in scored] + return source_name, results, raw_tracks + except Exception as exc: + logger.error( + f"[MultiSourceSearch] Search failed for {source_name}: {exc}", + exc_info=True, + ) + return source_name, [], [] + + +def search_all_sources(query: TrackQuery, + sources: List[Tuple[str, Any]], + clean_title: Optional[str] = None, + max_workers: int = 3) -> MultiSourceResult: + """Run a parallel metadata search across every source in ``sources``. + + Args: + query: TrackQuery describing the library track we want to match. + sources: List of ``(name, client)`` pairs. Each client must + implement ``search_tracks(query: str, limit: int) -> List[Track]`` + where each Track has ``.id``, ``.name``, ``.artists`` (list), + ``.album``, ``.duration_ms``, ``.image_url`` attributes. + All five primary metadata clients (Spotify / iTunes / + Deezer / Discogs / Hydrabase) satisfy this contract. + clean_title: Optional pre-cleaned track title (e.g. with + "(Remastered)" / "(Single Version)" suffixes stripped). + Defaults to ``query.title`` if not supplied. + max_workers: ThreadPoolExecutor pool size. Default 3 matches + the redownload endpoint's pre-extraction default — bumping + higher rate-limits on slower sources without speeding up + the slowest source's response. + + Returns: + MultiSourceResult with per-source results + cross-source best match. + """ + if clean_title is None: + clean_title = query.title + + if not sources: + return MultiSourceResult() + + metadata_results: Dict[str, List[dict]] = {} + raw_tracks: Dict[str, List[Any]] = {} + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = { + pool.submit(_search_one_source, name, client, query, clean_title): name + for name, client in sources + } + for future in as_completed(futures): + source_name, results, raws = future.result() + metadata_results[source_name] = results + raw_tracks[source_name] = raws + + best_match: Optional[dict] = None + for source, results in metadata_results.items(): + if results: + top = results[0] + if best_match is None or top['match_score'] > best_match['score']: + best_match = { + 'source': source, + 'index': 0, + 'score': top['match_score'], + } + + return MultiSourceResult( + metadata_results=metadata_results, + raw_tracks=raw_tracks, + best_match=best_match, + ) diff --git a/core/metadata/registry.py b/core/metadata/registry.py index 71206cfe..d2dc86f7 100644 --- a/core/metadata/registry.py +++ b/core/metadata/registry.py @@ -9,6 +9,7 @@ from __future__ import annotations import threading import hashlib +import time from typing import Any, Callable, Dict, Optional from utils.logging_config import get_logger @@ -277,8 +278,8 @@ def get_hydrabase_client(allow_fallback: bool = True, require_enabled: bool = Tr if client and client.is_connected(): if not require_enabled or bool(_dev_mode_enabled_provider()): return client - except Exception: - pass + except Exception as e: + logger.debug("hydrabase client lookup: %s", e) if allow_fallback: return get_itunes_client() @@ -341,6 +342,44 @@ def get_primary_client( ) +def get_primary_source_status( + *, + spotify_client_factory: Optional[MetadataClientFactory] = None, + itunes_client_factory: Optional[MetadataClientFactory] = None, + deezer_client_factory: Optional[MetadataClientFactory] = None, + discogs_client_factory: Optional[MetadataClientFactory] = None, +) -> Dict[str, Any]: + """Return a generic status snapshot for the active primary metadata source.""" + source = _get_config_value("metadata.fallback_source", "deezer") or "deezer" + started = time.time() + connected = False + + try: + client = get_client_for_source( + source, + spotify_client_factory=spotify_client_factory, + itunes_client_factory=itunes_client_factory, + deezer_client_factory=deezer_client_factory, + discogs_client_factory=discogs_client_factory, + ) + if source == "spotify": + connected = bool(client and client.is_spotify_authenticated()) + elif source == "hydrabase": + connected = bool(client and (client.is_connected() if hasattr(client, "is_connected") else client.is_authenticated())) + elif client is not None and hasattr(client, "is_authenticated"): + connected = bool(client.is_authenticated()) + else: + connected = client is not None + except Exception: + connected = False + + return { + "source": source, + "connected": connected, + "response_time": round((time.time() - started) * 1000, 1), + } + + def get_client_for_source( source: str, *, @@ -355,8 +394,8 @@ def get_client_for_source( client = get_spotify_client(client_factory=spotify_client_factory) if client and client.is_spotify_authenticated(): return client - except Exception: - pass + except Exception as e: + logger.debug("spotify client get_for_source: %s", e) return None if source == "deezer": diff --git a/core/metadata/source.py b/core/metadata/source.py index 478eeb02..bbe8dab9 100644 --- a/core/metadata/source.py +++ b/core/metadata/source.py @@ -124,12 +124,14 @@ SOURCE_TAG_CONFIG = { "AUDIODB_TRACK_ID": "audiodb.tags.track_id", "TIDAL_TRACK_ID": "tidal.tags.track_id", "TIDAL_ARTIST_ID": "tidal.tags.artist_id", + "HIFI_TRACK_ID": "hifi.tags.track_id", + "HIFI_ARTIST_ID": "hifi.tags.artist_id", "QOBUZ_TRACK_ID": "qobuz.tags.track_id", "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", "GENIUS_TRACK_ID": "genius.tags.track_id", } -DEFAULT_SOURCE_ORDER = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] +DEFAULT_SOURCE_ORDER = ["musicbrainz", "deezer", "audiodb", "tidal", "hifi", "qobuz", "lastfm", "genius"] ID3_TAG_MAP = { "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), @@ -253,7 +255,8 @@ def _process_musicbrainz_source(pp: dict, metadata: dict, cfg, runtime, track_ti album_name_for_mb = metadata.get("album", "") if album_name_for_mb: artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() - rc_key_norm = (normalize_album_cache_key(album_name_for_mb), artist_key) + normalized_album_key = normalize_album_cache_key(album_name_for_mb) + rc_key_norm = (normalized_album_key, artist_key) rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) release_mbid = None with mb_release_cache_lock: @@ -263,12 +266,38 @@ def _process_musicbrainz_source(pp: dict, metadata: dict, cfg, runtime, track_ti if cached: release_mbid = cached else: - rc_result = _call_source_lookup("MusicBrainz release", mb_service.match_release, album_name_for_mb, artist_name) - if rc_result and rc_result.get("mbid"): - release_mbid = rc_result["mbid"] + # Persistent cache check BEFORE the live MB lookup. If a + # previous SoulSync run already resolved this album's + # release MBID, reuse it — guarantees every track of the + # same album gets the SAME MUSICBRAINZ_ALBUMID tag, even + # across server restarts and after the in-memory bounded + # cache evicts the entry. Strictly additive: any failure + # in the persistent lookup falls through to the live MB + # query exactly as today. + try: + from core.metadata import album_mbid_cache as _persisted_cache + persisted = _persisted_cache.lookup(normalized_album_key, artist_key) + except Exception: + persisted = None + + if persisted: + release_mbid = persisted + else: + rc_result = _call_source_lookup("MusicBrainz release", mb_service.match_release, album_name_for_mb, artist_name) + if rc_result and rc_result.get("mbid"): + release_mbid = rc_result["mbid"] + if release_mbid: _bounded_cache_set(mb_release_cache, rc_key_norm, release_mbid, _MB_RELEASE_CACHE_MAX_ENTRIES) _bounded_cache_set(mb_release_cache, rc_key_exact, release_mbid, _MB_RELEASE_CACHE_MAX_ENTRIES) + # Also persist for future SoulSync runs. Defensive + # try/except so a DB write failure can't block the + # in-memory store + tag write that follow. + try: + from core.metadata import album_mbid_cache as _persisted_cache + _persisted_cache.record(normalized_album_key, artist_key, release_mbid) + except Exception as e: + logger.debug("MBID cache persist failed: %s", e) pp["release_mbid"] = release_mbid or "" if pp["release_mbid"]: pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] @@ -452,6 +481,9 @@ def _process_tidal_source(pp: dict, metadata: dict, cfg, runtime, track_title: s td_details = _call_source_lookup("Tidal track details", tidal_client.get_track, str(td_track_id)) if td_details: pp["tidal_isrc"] = td_details.get("isrc") + td_bpm = td_details.get("bpm") + if td_bpm and td_bpm > 0: + pp["tidal_bpm"] = td_bpm td_copyright = td_details.get("copyright") if isinstance(td_copyright, dict): td_copyright = td_copyright.get("text", td_copyright.get("name", "")) @@ -465,6 +497,47 @@ def _process_tidal_source(pp: dict, metadata: dict, cfg, runtime, track_title: s pp["release_year"] = td_release[:4] +def _process_hifi_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if cfg.get("hifi.embed_tags", True) is False: + return + if not track_title or not artist_name: + return + + hifi_client = getattr(runtime, "hifi_client", None) + if not hifi_client: + return + hifi_results = _call_source_lookup("HiFi track", hifi_client.search_tracks, track_title, artist_name) + if hifi_results and len(hifi_results) > 0: + hifi_track = hifi_results[0] + if _names_match(hifi_track.get("title", ""), track_title): + hifi_track_id = hifi_track.get("id") + if hifi_track_id: + pp["id_tags"]["HIFI_TRACK_ID"] = str(hifi_track_id) + hifi_artist_id = hifi_track.get("artist_id") + if hifi_artist_id: + pp["id_tags"]["HIFI_ARTIST_ID"] = str(hifi_artist_id) + if hifi_track_id: + hifi_details = _call_source_lookup("HiFi track details", hifi_client.get_track_info, hifi_track_id) + if hifi_details: + hifi_isrc = hifi_details.get("isrc") + if hifi_isrc: + pp["hifi_isrc"] = hifi_isrc + hifi_bpm = hifi_details.get("bpm") + if hifi_bpm and hifi_bpm > 0: + pp["hifi_bpm"] = hifi_bpm + hifi_copyright = hifi_details.get("copyright") + if hifi_copyright: + pp["hifi_copyright"] = hifi_copyright + if not pp["release_year"]: + hifi_album_id = hifi_track.get("album_id") + if hifi_album_id: + hifi_album = _call_source_lookup("HiFi album", hifi_client.get_album, hifi_album_id) + if hifi_album: + hifi_release = str(hifi_album.get("release_date", "") or "") + if len(hifi_release) >= 4 and hifi_release[:4].isdigit(): + pp["release_year"] = hifi_release[:4] + + def _process_qobuz_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: if cfg.get("qobuz.embed_tags", True) is False: return @@ -572,6 +645,8 @@ def _process_source_enrichment(source_name: str, pp: dict, metadata: dict, cfg, _process_audiodb_source(pp, metadata, cfg, runtime, track_title, artist_name) elif source_name == "tidal": _process_tidal_source(pp, metadata, cfg, runtime, track_title, artist_name) + elif source_name == "hifi": + _process_hifi_source(pp, metadata, cfg, runtime, track_title, artist_name) elif source_name == "qobuz": _process_qobuz_source(pp, metadata, cfg, runtime, track_title, artist_name) elif source_name == "lastfm": @@ -634,15 +709,23 @@ def _write_embedded_metadata(audio_file, metadata: dict, pp: dict, cfg, symbols) audio_file["\xa9day"] = [release_year] logger.info("Date tag: %s", release_year) - if _tag_enabled(cfg, "deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: - bpm_int = int(pp["deezer_bpm"]) + bpm_candidates = [] + if pp["deezer_bpm"] and pp["deezer_bpm"] > 0 and _tag_enabled(cfg, "deezer.tags.bpm"): + bpm_candidates.append(("Deezer", pp["deezer_bpm"])) + if pp["tidal_bpm"] and pp["tidal_bpm"] > 0 and _tag_enabled(cfg, "tidal.tags.bpm"): + bpm_candidates.append(("Tidal", pp["tidal_bpm"])) + if pp["hifi_bpm"] and pp["hifi_bpm"] > 0 and _tag_enabled(cfg, "hifi.tags.bpm"): + bpm_candidates.append(("HiFi", pp["hifi_bpm"])) + if bpm_candidates: + bpm_source, bpm_val = bpm_candidates[0] + bpm_int = int(bpm_val) if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) elif is_vorbis_like(audio_file, symbols): audio_file["BPM"] = [str(bpm_int)] elif isinstance(audio_file, symbols.MP4): audio_file["tmpo"] = [bpm_int] - logger.info("BPM: %s", bpm_int) + logger.info("BPM (%s): %s", bpm_source, bpm_int) if _tag_enabled(cfg, "audiodb.tags.mood") and pp["audiodb_mood"]: if isinstance(audio_file.tags, symbols.ID3): @@ -699,6 +782,8 @@ def _write_embedded_metadata(audio_file, metadata: dict, pp: dict, cfg, symbols) isrc_candidates.append(("Deezer", pp["deezer_isrc"])) if pp["tidal_isrc"] and _tag_enabled(cfg, "tidal.tags.isrc"): isrc_candidates.append(("Tidal", pp["tidal_isrc"])) + if pp["hifi_isrc"] and _tag_enabled(cfg, "hifi.tags.isrc"): + isrc_candidates.append(("HiFi", pp["hifi_isrc"])) if pp["qobuz_isrc"] and _tag_enabled(cfg, "qobuz.tags.isrc"): isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) if isrc_candidates: @@ -716,6 +801,8 @@ def _write_embedded_metadata(audio_file, metadata: dict, pp: dict, cfg, symbols) copyright_candidates.append(("Tidal", pp["tidal_copyright"])) if pp["qobuz_copyright"] and _tag_enabled(cfg, "qobuz.tags.copyright"): copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) + if pp["hifi_copyright"] and _tag_enabled(cfg, "hifi.tags.copyright"): + copyright_candidates.append(("HiFi", pp["hifi_copyright"])) if copyright_candidates: copyright_source, final_copyright = copyright_candidates[0] if isinstance(audio_file.tags, symbols.ID3): @@ -872,8 +959,8 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di resolved = itunes_client.resolve_primary_artist(artist_id) if resolved and resolved != raw_album_artist: raw_album_artist = resolved - except Exception: - pass + except Exception as e: + logger.debug("itunes primary artist resolve failed: %s", e) metadata["album_artist"] = raw_album_artist if album_info.get("is_album"): @@ -963,11 +1050,15 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N "isrc": None, "deezer_bpm": None, "deezer_isrc": None, + "tidal_bpm": None, + "hifi_bpm": None, + "hifi_copyright": None, "audiodb_mood": None, "audiodb_style": None, "audiodb_genre": None, "tidal_isrc": None, "tidal_copyright": None, + "hifi_isrc": None, "qobuz_isrc": None, "qobuz_copyright": None, "qobuz_label": None, @@ -981,12 +1072,46 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N if not isinstance(source_order, list) or not source_order: source_order = DEFAULT_SOURCE_ORDER + # If this download came from HiFi, use cached metadata from the download + # pipeline instead of re-searching the HiFi API. + original_search = get_import_original_search(context) + cached_meta = original_search.get("_source_metadata") or {} + if cached_meta.get("source") == "hifi": + if _tag_enabled(cfg, "hifi.embed_tags"): + if cfg.get("hifi.tags.track_id", True) and cached_meta.get("track_id"): + pp["id_tags"]["HIFI_TRACK_ID"] = str(cached_meta["track_id"]) + if cfg.get("hifi.tags.artist_id", True) and cached_meta.get("artist_id"): + pp["id_tags"]["HIFI_ARTIST_ID"] = str(cached_meta["artist_id"]) + if cfg.get("hifi.tags.isrc", True) and cached_meta.get("isrc"): + pp["hifi_isrc"] = cached_meta["isrc"] + if cfg.get("hifi.tags.bpm", True) and cached_meta.get("bpm"): + pp["hifi_bpm"] = cached_meta["bpm"] + if cfg.get("hifi.tags.copyright", True) and cached_meta.get("copyright"): + pp["hifi_copyright"] = cached_meta["copyright"] + source_order = [s for s in source_order if s != "hifi"] + + # If this download came from Tidal, use cached metadata from the download + # pipeline instead of re-searching the Tidal API. + if cached_meta.get("source") == "tidal": + if _tag_enabled(cfg, "tidal.embed_tags"): + if cfg.get("tidal.tags.track_id", True) and cached_meta.get("track_id"): + pp["id_tags"]["TIDAL_TRACK_ID"] = str(cached_meta["track_id"]) + if cfg.get("tidal.tags.artist_id", True) and cached_meta.get("artist_id"): + pp["id_tags"]["TIDAL_ARTIST_ID"] = str(cached_meta["artist_id"]) + if cfg.get("tidal.tags.isrc", True) and cached_meta.get("isrc"): + pp["tidal_isrc"] = cached_meta["isrc"] + if cfg.get("tidal.tags.bpm", True) and cached_meta.get("bpm"): + pp["tidal_bpm"] = cached_meta["bpm"] + if cfg.get("tidal.tags.copyright", True) and cached_meta.get("copyright"): + pp["tidal_copyright"] = cached_meta["copyright"] + source_order = [s for s in source_order if s != "tidal"] + db = get_database() for source_name in source_order: _process_source_enrichment(source_name, pp, metadata, cfg, runtime, track_title, artist_name) - if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: + if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["tidal_bpm"] and not pp["hifi_bpm"] and not pp["hifi_copyright"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: return release_year = _write_embedded_metadata(audio_file, metadata, pp, cfg, symbols) @@ -995,5 +1120,24 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N metadata["musicbrainz_release_id"] = release_id _update_album_year_in_database(db, metadata, release_year) + # Expose the final ID tag set + per-source values back to the + # import context so downstream side-effects (notably + # ``record_download_provenance``) can persist them to the + # ``track_downloads`` table without re-collecting. Without this, + # the watchlist scanner would have to wait for the async + # enrichment workers to backfill ``tracks.spotify_track_id`` etc. + # before recognizing freshly downloaded files. + if isinstance(context, dict): + try: + context["_embedded_id_tags"] = dict(pp.get("id_tags") or {}) + isrc_value = ( + pp.get("isrc") or pp.get("deezer_isrc") or pp.get("tidal_isrc") + or pp.get("hifi_isrc") or pp.get("qobuz_isrc") + ) + if isrc_value: + context["_isrc"] = str(isrc_value) + except Exception as e: + logger.debug("context isrc copy failed: %s", e) + except Exception as exc: logger.error("Error embedding source IDs (non-fatal): %s", exc) diff --git a/core/metadata/status.py b/core/metadata/status.py new file mode 100644 index 00000000..9e3fdffb --- /dev/null +++ b/core/metadata/status.py @@ -0,0 +1,179 @@ +"""Cached metadata-provider and Spotify status snapshots.""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Dict, Optional + +from utils.logging_config import get_logger + +from core.metadata.registry import get_primary_source_status + +logger = get_logger("metadata.status") + +METADATA_SOURCE_STATUS_TTL = 120 + +_UNSET = object() + +_status_lock = threading.RLock() +_metadata_source_status_cache: Dict[str, Any] = { + "source": "deezer", + "connected": False, + "response_time": 0, +} +_metadata_source_status_timestamp = 0.0 +_spotify_status_cache: Dict[str, Any] = { + "connected": False, + "authenticated": False, + "rate_limited": False, + "rate_limit": None, + "post_ban_cooldown": None, +} +_spotify_status_timestamp = 0.0 +_spotify_status_initialized = False +_spotify_rate_limit_expires_at = 0.0 +_spotify_post_ban_cooldown_expires_at = 0.0 + + +def invalidate_metadata_status_caches() -> None: + """Mark the cached metadata-source snapshot stale.""" + global _metadata_source_status_timestamp + with _status_lock: + _metadata_source_status_timestamp = 0.0 + + +def publish_spotify_status( + *, + connected: Any = _UNSET, + authenticated: Any = _UNSET, + rate_limited: Any = _UNSET, + rate_limit: Any = _UNSET, + post_ban_cooldown: Any = _UNSET, +) -> Dict[str, Any]: + """Update the cached Spotify status snapshot from an event.""" + global _spotify_status_timestamp, _spotify_status_initialized + global _spotify_rate_limit_expires_at, _spotify_post_ban_cooldown_expires_at + + with _status_lock: + if connected is not _UNSET: + _spotify_status_cache["connected"] = connected + if authenticated is not _UNSET: + _spotify_status_cache["authenticated"] = authenticated + if rate_limited is not _UNSET: + _spotify_status_cache["rate_limited"] = rate_limited + if rate_limit is not _UNSET: + _spotify_status_cache["rate_limit"] = rate_limit + if rate_limit and isinstance(rate_limit, dict): + _spotify_rate_limit_expires_at = float(rate_limit.get("expires_at") or 0.0) + else: + _spotify_rate_limit_expires_at = 0.0 + if post_ban_cooldown is not _UNSET: + _spotify_status_cache["post_ban_cooldown"] = post_ban_cooldown + if post_ban_cooldown is not None: + _spotify_post_ban_cooldown_expires_at = time.time() + max(0, float(post_ban_cooldown)) + else: + _spotify_post_ban_cooldown_expires_at = 0.0 + _spotify_status_timestamp = time.time() + _spotify_status_initialized = True + _normalize_spotify_status_locked(_spotify_status_timestamp) + return dict(_spotify_status_cache) + + +def refresh_spotify_status_from_client(spotify_client: Optional[Any]) -> Dict[str, Any]: + """Probe Spotify once to seed the cache when no event has populated it yet.""" + if spotify_client is None: + with _status_lock: + return dict(_spotify_status_cache) + + try: + is_rate_limited = spotify_client.is_rate_limited() if spotify_client else False + rate_limit_info = spotify_client.get_rate_limit_info() if (spotify_client and is_rate_limited) else None + cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0 + authenticated = spotify_client.is_spotify_authenticated() if spotify_client else False + except Exception as exc: + logger.debug("Spotify status probe failed: %s", exc) + authenticated = False + is_rate_limited = False + rate_limit_info = None + cooldown_remaining = 0 + + return publish_spotify_status( + connected=authenticated, + authenticated=authenticated, + rate_limited=is_rate_limited, + rate_limit=rate_limit_info, + post_ban_cooldown=cooldown_remaining if cooldown_remaining > 0 else None, + ) + + +def get_metadata_source_status() -> Dict[str, Any]: + """Return a cached snapshot for the active primary metadata source.""" + global _metadata_source_status_timestamp + + current_time = time.time() + with _status_lock: + if _metadata_source_status_timestamp and current_time - _metadata_source_status_timestamp <= METADATA_SOURCE_STATUS_TTL: + return dict(_metadata_source_status_cache) + + try: + status_data = get_primary_source_status() + except Exception as exc: + logger.debug("Metadata source status refresh failed: %s", exc) + status_data = None + + if status_data: + with _status_lock: + _metadata_source_status_cache.update(status_data) + _metadata_source_status_timestamp = current_time + + with _status_lock: + return dict(_metadata_source_status_cache) + + +def get_spotify_status(spotify_client: Optional[Any] = None) -> Dict[str, Any]: + """Return a cached Spotify-specific status snapshot.""" + with _status_lock: + if _spotify_status_initialized: + _normalize_spotify_status_locked(time.time()) + return dict(_spotify_status_cache) + + return refresh_spotify_status_from_client(spotify_client) + + +def _normalize_spotify_status_locked(current_time: float) -> None: + """Update derived Spotify status fields and clear expired ban state.""" + global _spotify_rate_limit_expires_at, _spotify_post_ban_cooldown_expires_at + + rate_limit = _spotify_status_cache.get("rate_limit") + if _spotify_status_cache.get("rate_limited") and rate_limit and isinstance(rate_limit, dict): + expires_at = float(rate_limit.get("expires_at") or 0.0) + if expires_at > 0: + remaining = int(max(0, expires_at - current_time)) + if remaining > 0: + _spotify_status_cache["rate_limit"] = {**rate_limit, "remaining_seconds": remaining} + _spotify_rate_limit_expires_at = expires_at + else: + _spotify_status_cache["rate_limited"] = False + _spotify_status_cache["rate_limit"] = None + _spotify_rate_limit_expires_at = 0.0 + elif _spotify_rate_limit_expires_at and current_time >= _spotify_rate_limit_expires_at: + _spotify_status_cache["rate_limited"] = False + _spotify_status_cache["rate_limit"] = None + _spotify_rate_limit_expires_at = 0.0 + + if _spotify_post_ban_cooldown_expires_at > 0: + remaining = int(max(0, _spotify_post_ban_cooldown_expires_at - current_time)) + if remaining > 0: + _spotify_status_cache["post_ban_cooldown"] = remaining + else: + _spotify_status_cache["post_ban_cooldown"] = None + _spotify_post_ban_cooldown_expires_at = 0.0 + + +def get_status_snapshot(spotify_client: Optional[Any] = None) -> Dict[str, Any]: + """Return the combined metadata-provider status snapshot.""" + return { + "metadata_source": get_metadata_source_status(), + "spotify": get_spotify_status(spotify_client=spotify_client), + } diff --git a/core/metadata/types.py b/core/metadata/types.py new file mode 100644 index 00000000..4ed01ff4 --- /dev/null +++ b/core/metadata/types.py @@ -0,0 +1,618 @@ +"""Canonical typed dataclasses for metadata across all providers. + +The metadata pipeline historically grew organically: each new provider +(Spotify → iTunes → Deezer → Tidal → Qobuz → MusicBrainz → AudioDB → +Discogs → Hydrabase) returns its own response shape, and consumer code +defensively extracts every field via fallback chains: + + _extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', + 'release_id', default=album_id) + +That pattern works but is brittle: each new provider adds more keys to +chase, each consumer re-runs the same defensive logic, and there's no +contract about what shape any given consumer can trust. + +This module is the canonical contract. Every provider produces these +types via a single ``from__dict()`` classmethod. Every +consumer accepts these types and trusts the fields. Field names are +provider-neutral (``release_date`` not ``releaseDate``, +``image_url`` not ``artworkUrl100``). + +This is the foundation PR. It only DEFINES the contract and provides +the converters; no consumer is migrated in this PR. Future PRs each +migrate one consumer to accept ``Album`` / ``Track`` / ``Artist`` +instead of raw dicts. + +The ``Album`` / ``Track`` / ``Artist`` symbols also re-export from +``core.itunes_client`` for backward compatibility — existing callers +don't need to change anything. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +# --------------------------------------------------------------------------- +# Helpers shared by converters +# --------------------------------------------------------------------------- + + +def _str(value: Any, default: str = '') -> str: + """Coerce to non-None str, never None.""" + if value is None: + return default + return str(value) + + +def _int(value: Any, default: int = 0) -> int: + """Coerce to int, default on parse failure.""" + if value is None or value == '': + return default + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _strip_discogs_disambiguation(name: str) -> str: + """Discogs appends ``(N)`` to artist names when there are multiple + artists with the same name. Strip so cross-provider matches work.""" + return re.sub(r'\s*\(\d+\)$', '', name or '').strip() + + +def _itunes_artwork(url: Optional[str]) -> Optional[str]: + """iTunes serves cover art at any size by template substitution. + Always upgrade ``100x100bb`` → ``3000x3000bb`` for highest quality.""" + if not url: + return None + return url.replace('100x100bb', '3000x3000bb') + + +# --------------------------------------------------------------------------- +# Album +# --------------------------------------------------------------------------- + + +@dataclass +class Album: + """Provider-neutral album. + + Required fields are guaranteed to be set by every converter. Optional + fields are explicit ``Optional[...]`` so consumers know they may be + None / empty. Source-specific raw IDs that don't fit the typed schema + can be stashed in ``external_ids`` (provider name → id string). + """ + + id: str # Source-native id, always set + name: str # Album title, always set + artists: List[str] # Display names, may be ['Unknown Artist'] + release_date: str # ISO 'YYYY' or 'YYYY-MM-DD' or '' when unknown + total_tracks: int # 0 when unknown + album_type: str # 'album' / 'single' / 'ep' / 'compilation' + + # Optional but commonly populated + image_url: Optional[str] = None # Highest-quality cover URL + artist_id: Optional[str] = None # Primary artist's source-native id + genres: List[str] = field(default_factory=list) + label: Optional[str] = None # Record label / publisher + barcode: Optional[str] = None # UPC/EAN — Discogs/MusicBrainz only + + # Source provenance + source: str = '' # 'spotify' / 'itunes' / etc — set by converter + external_ids: Dict[str, str] = field(default_factory=dict) + external_urls: Dict[str, str] = field(default_factory=dict) + + # ------------------------------------------------------------------ + # Per-source converters. Each one is the SINGLE source of truth for + # how that provider's response maps to the canonical Album. Adding + # a new provider = adding one more converter here. Consumer code + # never needs to know any provider's wire shape. + # ------------------------------------------------------------------ + + @classmethod + def from_spotify_dict(cls, raw: Dict[str, Any]) -> 'Album': + """Spotify Web API ``/albums/{id}`` response shape.""" + artists_raw = raw.get('artists') or [] + artist_names = [_str(a.get('name')) for a in artists_raw + if isinstance(a, dict) and a.get('name')] + primary_artist_id = '' + if artists_raw and isinstance(artists_raw[0], dict): + primary_artist_id = _str(artists_raw[0].get('id')) + + images = raw.get('images') or [] + image_url = None + if images and isinstance(images[0], dict): + image_url = _str(images[0].get('url')) or None + + external_ids = {} + if raw.get('id'): + external_ids['spotify'] = _str(raw['id']) + upc = (raw.get('external_ids') or {}).get('upc') + if upc: + external_ids['upc'] = _str(upc) + + external_urls = {} + sp_url = (raw.get('external_urls') or {}).get('spotify') + if sp_url: + external_urls['spotify'] = _str(sp_url) + + return cls( + id=_str(raw.get('id')), + name=_str(raw.get('name')), + artists=artist_names or ['Unknown Artist'], + release_date=_str(raw.get('release_date')), + total_tracks=_int(raw.get('total_tracks')), + album_type=_str(raw.get('album_type'), default='album'), + image_url=image_url, + artist_id=primary_artist_id or None, + genres=list(raw.get('genres') or []), + label=_str(raw.get('label')) or None, + barcode=external_ids.get('upc'), + source='spotify', + external_ids=external_ids, + external_urls=external_urls, + ) + + @classmethod + def from_itunes_dict(cls, raw: Dict[str, Any]) -> 'Album': + """iTunes Search API album response shape (`collectionType=Album`).""" + track_count = _int(raw.get('trackCount')) + + # iTunes doesn't tag album type; infer from track count + collectionType. + collection_type = _str(raw.get('collectionType'), default='Album') + if 'compilation' in collection_type.lower(): + album_type = 'compilation' + elif track_count <= 3: + album_type = 'single' + elif track_count <= 6: + album_type = 'ep' + else: + album_type = 'album' + + artist_id = _str(raw.get('artistId')) or None + external_ids = {} + if raw.get('collectionId'): + external_ids['itunes'] = _str(raw['collectionId']) + if artist_id: + external_ids['itunes_artist'] = artist_id + + external_urls = {} + if raw.get('collectionViewUrl'): + external_urls['itunes'] = _str(raw['collectionViewUrl']) + + # Strip iTunes "(Single)" / "(EP)" / "(Deluxe)" suffixes from name + # the same way the existing _clean_itunes_album_name helper does. + name = _str(raw.get('collectionName')) + name = re.sub(r'\s*[-(]\s*(Single|EP)\s*[)]?$', '', name, flags=re.IGNORECASE).strip() + + release_date = _str(raw.get('releaseDate')) + if release_date and 'T' in release_date: + release_date = release_date.split('T', 1)[0] + + primary_genre = _str(raw.get('primaryGenreName')) + return cls( + id=_str(raw.get('collectionId')), + name=name, + artists=[_str(raw.get('artistName'), default='Unknown Artist')], + release_date=release_date, + total_tracks=track_count, + album_type=album_type, + image_url=_itunes_artwork(raw.get('artworkUrl100')), + artist_id=artist_id, + genres=[primary_genre] if primary_genre else [], + source='itunes', + external_ids=external_ids, + external_urls=external_urls, + ) + + @classmethod + def from_deezer_dict(cls, raw: Dict[str, Any]) -> 'Album': + """Deezer API ``/album/{id}`` response shape.""" + artist = raw.get('artist') or {} + artist_name = _str(artist.get('name'), default='Unknown Artist') if isinstance(artist, dict) else _str(artist) or 'Unknown Artist' + artist_id = _str(artist.get('id')) if isinstance(artist, dict) else '' + + # Deezer cover URLs come in size suffixes (cover_xl, cover_big, + # cover_medium, cover_small). Prefer xl. + image_url = ( + _str(raw.get('cover_xl')) + or _str(raw.get('cover_big')) + or _str(raw.get('cover_medium')) + or _str(raw.get('cover')) + or None + ) + + record_type = _str(raw.get('record_type'), default='album').lower() + album_type = {'single': 'single', 'ep': 'ep'}.get(record_type, 'album') + + external_ids = {} + if raw.get('id'): + external_ids['deezer'] = _str(raw['id']) + if raw.get('upc'): + external_ids['upc'] = _str(raw['upc']) + + external_urls = {} + if raw.get('link'): + external_urls['deezer'] = _str(raw['link']) + + return cls( + id=_str(raw.get('id')), + name=_str(raw.get('title')), + artists=[artist_name], + release_date=_str(raw.get('release_date')), + total_tracks=_int(raw.get('nb_tracks')), + album_type=album_type, + image_url=image_url, + artist_id=artist_id or None, + genres=[g.get('name', '') for g in (raw.get('genres', {}) or {}).get('data', []) + if isinstance(g, dict) and g.get('name')], + label=_str(raw.get('label')) or None, + barcode=external_ids.get('upc'), + source='deezer', + external_ids=external_ids, + external_urls=external_urls, + ) + + @classmethod + def from_discogs_dict(cls, raw: Dict[str, Any]) -> 'Album': + """Discogs API ``/releases/{id}`` response shape.""" + artists_raw = raw.get('artists') or [] + artist_names = [] + primary_artist_id = '' + for a in artists_raw: + if not isinstance(a, dict): + continue + name = _strip_discogs_disambiguation(_str(a.get('name'))) + if name: + artist_names.append(name) + if not primary_artist_id and a.get('id'): + primary_artist_id = _str(a['id']) + + images = raw.get('images') or [] + image_url = None + if images and isinstance(images[0], dict): + image_url = _str(images[0].get('uri') or images[0].get('uri150')) or None + + # Discogs `tracklist` is the source of total_tracks. + tracklist = raw.get('tracklist') or [] + total_tracks = sum(1 for t in tracklist if isinstance(t, dict) + and t.get('type_') == 'track') + if not total_tracks: + total_tracks = len(tracklist) + + labels = raw.get('labels') or [] + label_name = '' + if labels and isinstance(labels[0], dict): + label_name = _str(labels[0].get('name')) + + external_ids = {} + if raw.get('id'): + external_ids['discogs'] = _str(raw['id']) + # Discogs `identifiers` array can include barcode entries + for ident in raw.get('identifiers', []) or []: + if isinstance(ident, dict) and ident.get('type', '').lower() == 'barcode': + bc = _str(ident.get('value')).strip() + if bc: + external_ids['barcode'] = bc + break + + external_urls = {} + if raw.get('uri'): + external_urls['discogs'] = _str(raw['uri']) + + year = raw.get('year') + release_date = str(year) if year and _int(year) > 0 else '' + + return cls( + id=_str(raw.get('id')), + name=_str(raw.get('title')), + artists=artist_names or ['Unknown Artist'], + release_date=release_date, + total_tracks=total_tracks, + album_type='album', # Discogs doesn't tag this; default to album + image_url=image_url, + artist_id=primary_artist_id or None, + genres=list(raw.get('genres') or []) + list(raw.get('styles') or []), + label=label_name or None, + barcode=external_ids.get('barcode'), + source='discogs', + external_ids=external_ids, + external_urls=external_urls, + ) + + @classmethod + def from_musicbrainz_dict(cls, raw: Dict[str, Any]) -> 'Album': + """MusicBrainz ``/release/{mbid}`` response shape (release, not release-group).""" + artist_credit = raw.get('artist-credit') or [] + artist_names = [] + primary_artist_id = '' + for credit in artist_credit: + if isinstance(credit, dict) and 'artist' in credit: + name = _str(credit['artist'].get('name')) + if name: + artist_names.append(name) + if not primary_artist_id and credit['artist'].get('id'): + primary_artist_id = _str(credit['artist']['id']) + + # Total tracks: sum across media (MB stores per-disc). + media = raw.get('media') or [] + total_tracks = sum(_int(m.get('track-count')) for m in media if isinstance(m, dict)) + + external_ids = {} + if raw.get('id'): + external_ids['musicbrainz'] = _str(raw['id']) + if raw.get('barcode'): + external_ids['barcode'] = _str(raw['barcode']) + + # MB `release-group` carries the album-level type (album/single/ep) + rg = raw.get('release-group') or {} + primary_type = _str(rg.get('primary-type'), default='Album').lower() + album_type = {'single': 'single', 'ep': 'ep'}.get(primary_type, 'album') + if rg.get('id'): + external_ids['musicbrainz_release_group'] = _str(rg['id']) + + labels = raw.get('label-info') or [] + label_name = '' + if labels and isinstance(labels[0], dict): + lbl = labels[0].get('label') or {} + label_name = _str(lbl.get('name')) + + return cls( + id=_str(raw.get('id')), + name=_str(raw.get('title')), + artists=artist_names or ['Unknown Artist'], + release_date=_str(raw.get('date')), + total_tracks=total_tracks, + album_type=album_type, + image_url=None, # MB doesn't serve cover art directly; CAA is separate + artist_id=primary_artist_id or None, + genres=[], # MB has tags but they're noisy; consumer can fetch separately + label=label_name or None, + barcode=external_ids.get('barcode'), + source='musicbrainz', + external_ids=external_ids, + external_urls={}, + ) + + @classmethod + def from_qobuz_dict(cls, raw: Dict[str, Any]) -> 'Album': + """Qobuz API ``album/get`` response shape.""" + artist = raw.get('artist') or {} + artist_name = _str(artist.get('name'), default='Unknown Artist') if isinstance(artist, dict) else _str(artist) or 'Unknown Artist' + artist_id = _str(artist.get('id')) if isinstance(artist, dict) else '' + + # Qobuz `image` is a dict with small/large/thumbnail variants. + image = raw.get('image') or {} + image_url = None + if isinstance(image, dict): + image_url = ( + _str(image.get('large')) + or _str(image.get('small')) + or _str(image.get('thumbnail')) + or None + ) + + external_ids = {} + if raw.get('id'): + external_ids['qobuz'] = _str(raw['id']) + if raw.get('upc'): + external_ids['upc'] = _str(raw['upc']) + + external_urls = {} + if raw.get('url'): + external_urls['qobuz'] = _str(raw['url']) + + # Qobuz exposes both `release_date_original` (vinyl/original + # press date) and `released_at` (digital release timestamp). + # Prefer the original date for cross-provider matching. + release_date = _str(raw.get('release_date_original') or raw.get('released_at')) + if release_date and 'T' in release_date: + release_date = release_date.split('T', 1)[0] + + genre = raw.get('genre') or {} + genre_name = _str(genre.get('name')) if isinstance(genre, dict) else _str(genre) + + label = raw.get('label') or {} + label_name = _str(label.get('name')) if isinstance(label, dict) else _str(label) + + return cls( + id=_str(raw.get('id')), + name=_str(raw.get('title')), + artists=[artist_name], + release_date=release_date, + total_tracks=_int(raw.get('tracks_count')), + album_type='album', # Qobuz doesn't tag this consistently + image_url=image_url, + artist_id=artist_id or None, + genres=[genre_name] if genre_name else [], + label=label_name or None, + barcode=external_ids.get('upc'), + source='qobuz', + external_ids=external_ids, + external_urls=external_urls, + ) + + @classmethod + def from_tidal_object(cls, obj: Any) -> 'Album': + """tidalapi ``Album`` object shape. + + Tidal goes through the ``tidalapi`` library which returns + Python objects, not raw dicts — so this converter is named + ``from_tidal_object`` to make the input contract explicit. + Duck-types attribute access so unit tests can pass simple + SimpleNamespace stand-ins.""" + artist = getattr(obj, 'artist', None) + artist_name = _str(getattr(artist, 'name', None), default='Unknown Artist') + artist_id = _str(getattr(artist, 'id', '')) if artist else '' + + # tidalapi exposes `image()` as a method that returns a URL at + # a given size. Try a sensible default size; fall back to the + # `picture` field (the raw image id) if the method's missing. + image_url = None + try: + if hasattr(obj, 'image') and callable(obj.image): + image_url = obj.image(640) or None + except Exception: + image_url = None + if not image_url: + picture = _str(getattr(obj, 'picture', '')) + if picture: + # Tidal CDN URL format + pic_path = picture.replace('-', '/') + image_url = f"https://resources.tidal.com/images/{pic_path}/640x640.jpg" + + release_date = '' + rd = getattr(obj, 'release_date', None) + if rd is not None: + release_date = _str(rd).split('T')[0] if 'T' in _str(rd) else _str(rd) + + external_ids = {} + if getattr(obj, 'id', None): + external_ids['tidal'] = _str(obj.id) + if getattr(obj, 'universal_product_number', None): + external_ids['upc'] = _str(obj.universal_product_number) + + return cls( + id=_str(getattr(obj, 'id', '')), + name=_str(getattr(obj, 'name', '')), + artists=[artist_name], + release_date=release_date, + total_tracks=_int(getattr(obj, 'num_tracks', 0)), + album_type=_str(getattr(obj, 'type', None), default='album').lower() or 'album', + image_url=image_url, + artist_id=artist_id or None, + genres=[], # tidalapi doesn't expose genres on Album + barcode=external_ids.get('upc'), + source='tidal', + external_ids=external_ids, + external_urls={}, + ) + + @classmethod + def from_hydrabase_dict(cls, raw: Dict[str, Any]) -> 'Album': + """Hydrabase metadata service response shape.""" + artists_raw = raw.get('artists') or [] + if isinstance(artists_raw, str): + artist_names = [artists_raw] + else: + artist_names = [] + for a in artists_raw: + if isinstance(a, dict): + name = _str(a.get('name')) + else: + name = _str(a) + if name: + artist_names.append(name) + + external_ids = {} + if raw.get('id'): + external_ids['hydrabase'] = _str(raw['id']) + if raw.get('soul_id'): + external_ids['soul'] = _str(raw['soul_id']) + + return cls( + id=_str(raw.get('id')), + name=_str(raw.get('name') or raw.get('title')), + artists=artist_names or ['Unknown Artist'], + release_date=_str(raw.get('release_date')), + total_tracks=_int(raw.get('total_tracks')), + album_type=_str(raw.get('album_type'), default='album'), + image_url=_str(raw.get('image_url') or raw.get('thumb_url')) or None, + artist_id=_str(raw.get('artist_id')) or None, + source='hydrabase', + external_ids=external_ids, + ) + + # ------------------------------------------------------------------ + # Consumer-side helpers + # ------------------------------------------------------------------ + + def to_context_dict(self) -> Dict[str, Any]: + """Return the canonical dict shape SoulSync's import / download + pipelines expect. This is the bridge between typed metadata and + the existing dict-passing internal API. Future PRs migrate + consumers off this dict shape and onto the typed Album directly, + at which point this helper becomes unnecessary.""" + primary_artist = self.artists[0] if self.artists else 'Unknown Artist' + artists_dicts = [{'name': name, 'id': self.artist_id if i == 0 else ''} + for i, name in enumerate(self.artists)] + images = [{'url': self.image_url}] if self.image_url else [] + + return { + 'id': self.id, + 'name': self.name, + 'artist': primary_artist, + 'artist_name': primary_artist, + 'artist_id': self.artist_id or '', + 'artists': artists_dicts, + 'image_url': self.image_url, + 'images': images, + 'release_date': self.release_date, + 'album_type': self.album_type, + 'total_tracks': self.total_tracks, + 'source': self.source, + 'genres': list(self.genres), + 'label': self.label or '', + 'barcode': self.barcode or '', + 'external_ids': dict(self.external_ids), + 'external_urls': dict(self.external_urls), + } + + +# --------------------------------------------------------------------------- +# Track and Artist — kept lighter for now. Future PRs flesh these out +# in the same per-source-converter pattern as Album. +# --------------------------------------------------------------------------- + + +@dataclass +class Track: + """Provider-neutral track. Required fields are always populated by + every provider's converter; optional fields may be None.""" + + id: str + name: str + artists: List[str] + album: str + duration_ms: int + + # Optional + track_number: Optional[int] = None + disc_number: Optional[int] = None + image_url: Optional[str] = None + release_date: Optional[str] = None + album_type: Optional[str] = None + total_tracks: Optional[int] = None + preview_url: Optional[str] = None + isrc: Optional[str] = None + popularity: int = 0 # Spotify-only; 0 elsewhere + + # Source provenance + source: str = '' + external_ids: Dict[str, str] = field(default_factory=dict) + external_urls: Dict[str, str] = field(default_factory=dict) + + +@dataclass +class Artist: + """Provider-neutral artist.""" + + id: str + name: str + + # Optional + image_url: Optional[str] = None + genres: List[str] = field(default_factory=list) + popularity: int = 0 # Spotify-only; 0 elsewhere + followers: int = 0 # Spotify-only; 0 elsewhere + + # Source provenance + source: str = '' + external_ids: Dict[str, str] = field(default_factory=dict) + external_urls: Dict[str, str] = field(default_factory=dict) + + +__all__ = ['Album', 'Track', 'Artist'] diff --git a/core/musicbrainz_worker.py b/core/musicbrainz_worker.py index 08e02248..fe6461fc 100644 --- a/core/musicbrainz_worker.py +++ b/core/musicbrainz_worker.py @@ -141,8 +141,8 @@ class MusicBrainzWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null id table resolve failed: %s", e) continue diff --git a/core/navidrome_client.py b/core/navidrome_client.py index 3730e36c..11995dff 100644 --- a/core/navidrome_client.py +++ b/core/navidrome_client.py @@ -2,33 +2,19 @@ import requests import hashlib import secrets from typing import List, Optional, Dict, Any -from dataclasses import dataclass from datetime import datetime import json from utils.logging_config import get_logger from config.settings import config_manager +# Shared dataclasses live in the neutral media_server package — every +# server client used to define a near-identical XTrackInfo / +# XPlaylistInfo. Lifted to one canonical type so consumers (matching +# engine, sync service) get a single import. +from core.media_server.types import TrackInfo, PlaylistInfo + logger = get_logger("navidrome_client") -@dataclass -class NavidromeTrackInfo: - id: str - title: str - artist: str - album: str - duration: int - track_number: Optional[int] = None - year: Optional[int] = None - rating: Optional[float] = None - -@dataclass -class NavidromePlaylistInfo: - id: str - title: str - description: Optional[str] - duration: int - leaf_count: int - tracks: List[NavidromeTrackInfo] class NavidromeArtist: """Wrapper class to mimic Plex artist object interface""" @@ -117,6 +103,14 @@ class NavidromeTrack: self.suffix = navidrome_data.get('suffix') # e.g. "flac", "mp3" self.bitRate = navidrome_data.get('bitRate') # e.g. 320 self.path = navidrome_data.get('path') # e.g. "/music/Artist/Album/track.flac" + # File size in bytes (Subsonic ) — powers the + # Library Disk Usage card on Stats. None when the server didn't + # report a size (rare but possible for streaming-only nodes). + _nv_size = navidrome_data.get('size') + try: + self.file_size = int(_nv_size) if _nv_size else None + except (TypeError, ValueError): + self.file_size = None self._album_id = navidrome_data.get('albumId', '') self._artist_id = navidrome_data.get('artistId', '') @@ -142,7 +136,10 @@ class NavidromeTrack: return self._client.get_album_by_id(self._album_id) return None -class NavidromeClient: +from core.media_server.contract import MediaServerClient + + +class NavidromeClient(MediaServerClient): def __init__(self): self.base_url: Optional[str] = None self.username: Optional[str] = None @@ -816,7 +813,7 @@ class NavidromeClient: return {} return {} - def get_all_playlists(self) -> List[NavidromePlaylistInfo]: + def get_all_playlists(self) -> List[PlaylistInfo]: """Get all playlists from Navidrome server""" if not self.ensure_connection(): return [] @@ -830,7 +827,7 @@ class NavidromeClient: playlists_data = response.get('playlists', {}).get('playlist', []) for playlist_data in playlists_data: - playlist_info = NavidromePlaylistInfo( + playlist_info = PlaylistInfo( id=playlist_data.get('id', ''), title=playlist_data.get('name', 'Unknown Playlist'), description=playlist_data.get('comment'), @@ -847,7 +844,7 @@ class NavidromeClient: logger.error(f"Error getting playlists from Navidrome: {e}") return [] - def get_playlist_by_name(self, name: str) -> Optional[NavidromePlaylistInfo]: + def get_playlist_by_name(self, name: str) -> Optional[PlaylistInfo]: """Get a specific playlist by name""" playlists = self.get_all_playlists() for playlist in playlists: @@ -925,8 +922,8 @@ class NavidromeClient: if target_playlist: self._make_request('deletePlaylist', {'id': target_playlist.id}) logger.info(f"Deleted existing backup playlist '{target_name}'") - except Exception: - pass # Target doesn't exist, which is fine + except Exception as e: + logger.debug("backup playlist precheck: %s", e) # Create new playlist with copied tracks try: @@ -968,7 +965,7 @@ class NavidromeClient: logger.error(f"Error getting tracks for playlist {playlist_id}: {e}") return [] - def get_playlists_by_name(self, name: str) -> List[NavidromePlaylistInfo]: + def get_playlists_by_name(self, name: str) -> List[PlaylistInfo]: """Get all playlists matching a specific name (case-insensitive)""" matches = [] playlists = self.get_all_playlists() @@ -1132,7 +1129,7 @@ class NavidromeClient: self._track_cache.clear() logger.info("Navidrome client cache cleared") - def search_tracks(self, title: str, artist: str, limit: int = 15) -> List[NavidromeTrackInfo]: + def search_tracks(self, title: str, artist: str, limit: int = 15) -> List[TrackInfo]: """Search for tracks using Navidrome search API""" if not self.ensure_connection(): logger.warning("Navidrome not connected. Cannot perform search.") @@ -1158,7 +1155,7 @@ class NavidromeClient: search_result = response.get('searchResult3', {}) for track_data in search_result.get('song', []): - track_info = NavidromeTrackInfo( + track_info = TrackInfo( id=track_data.get('id', ''), title=track_data.get('title', ''), artist=track_data.get('artist', ''), diff --git a/core/playlists/explorer.py b/core/playlists/explorer.py index 991f6915..7a4603d8 100644 --- a/core/playlists/explorer.py +++ b/core/playlists/explorer.py @@ -176,8 +176,8 @@ def playlist_explorer_build_tree(deps: PlaylistExplorerDeps): artist_image = images[0].get('url') if images else None elif hasattr(artist_info, 'image_url'): artist_image = artist_info.image_url - except Exception: - pass + except Exception as e: + logger.debug("artist image resolve: %s", e) else: # No pre-resolved ID — search by name try: @@ -224,8 +224,8 @@ def playlist_explorer_build_tree(deps: PlaylistExplorerDeps): cursor.execute("SELECT title FROM albums WHERE artist_id = ?", (ar['id'],)) for alb_row in cursor.fetchall(): owned_titles.add((alb_row['title'] or '').strip().lower()) - except Exception: - pass # Non-critical — owned badges just won't show + except Exception as e: + logger.debug("owned-titles lookup: %s", e) # Build release list releases = [] @@ -256,8 +256,8 @@ def playlist_explorer_build_tree(deps: PlaylistExplorerDeps): if cache and releases: try: cache.store_entity(source_name, 'artist_discography', cache_key, result) - except Exception: - pass + except Exception as e: + logger.debug("cache discography write: %s", e) return result diff --git a/core/plex_client.py b/core/plex_client.py index 43bbf8a8..e1d8a498 100644 --- a/core/plex_client.py +++ b/core/plex_client.py @@ -4,7 +4,6 @@ from plexapi.audio import Track as PlexTrack, Album as PlexAlbum, Artist as Plex from plexapi.playlist import Playlist as PlexPlaylist from plexapi.exceptions import PlexApiException, NotFound from typing import List, Optional, Dict, Any -from dataclasses import dataclass import requests from datetime import datetime, timedelta import re @@ -12,72 +11,36 @@ from utils.logging_config import get_logger from config.settings import config_manager import threading +# Shared dataclasses live in the neutral media_server package now — +# every server client used to define a near-identical XTrackInfo / +# XPlaylistInfo. Lifted to one canonical type so consumers (matching +# engine, sync service, etc.) get a single import. +from core.media_server.types import TrackInfo, PlaylistInfo +from core.media_server.contract import MediaServerClient + logger = get_logger("plex_client") -@dataclass -class PlexTrackInfo: - id: str - title: str - artist: str - album: str - duration: int - track_number: Optional[int] = None - year: Optional[int] = None - rating: Optional[float] = None - - @classmethod - def from_plex_track(cls, track: PlexTrack) -> 'PlexTrackInfo': - # Gracefully handle tracks that might be missing artist or album metadata in Plex - try: - artist_title = track.artist().title if track.artist() else "Unknown Artist" - except (NotFound, AttributeError): - artist_title = "Unknown Artist" - - try: - album_title = track.album().title if track.album() else "Unknown Album" - except (NotFound, AttributeError): - album_title = "Unknown Album" - return cls( - id=str(track.ratingKey), - title=track.title, - artist=artist_title, - album=album_title, - duration=track.duration, - track_number=track.trackNumber, - year=track.year, - rating=track.userRating - ) +# Sentinel value stored in the ``plex_music_library`` DB preference when +# the user picks "All Libraries (combined)". Detection is one string +# compare in ``_find_music_library`` / ``set_music_library_by_name``. +# Existing prefs (real library names) are unaffected — only this exact +# string flips the client into multi-section read mode. +ALL_LIBRARIES_SENTINEL = '__all_libraries__' -@dataclass -class PlexPlaylistInfo: - id: str - title: str - description: Optional[str] - duration: int - leaf_count: int - tracks: List[PlexTrackInfo] - - @classmethod - def from_plex_playlist(cls, playlist: PlexPlaylist) -> 'PlexPlaylistInfo': - tracks = [] - for item in playlist.items(): - if isinstance(item, PlexTrack): - tracks.append(PlexTrackInfo.from_plex_track(item)) - - return cls( - id=str(playlist.ratingKey), - title=playlist.title, - description=playlist.summary, - duration=playlist.duration, - leaf_count=playlist.leafCount, - tracks=tracks - ) -class PlexClient: +class PlexClient(MediaServerClient): def __init__(self): self.server: Optional[PlexServer] = None self.music_library: Optional[MusicSection] = None + # When True, read methods union across every music section under + # the connected token via ``server.library.search(libtype=...)`` + # instead of querying ``self.music_library`` only. Set by + # ``_find_music_library`` / ``set_music_library_by_name`` when the + # user's saved preference matches ``ALL_LIBRARIES_SENTINEL``. + # Write methods (genre / poster / metadata updates) are unaffected + # — they operate on Plex objects via ratingKey, section-agnostic. + self._all_libraries_mode = False self._connection_attempted = False self._is_connecting = False self._last_connection_check = 0 # Cache connection checks @@ -116,6 +79,7 @@ class PlexClient: logger.info("Resetting Plex connection state") self.server = None self.music_library = None + self._all_libraries_mode = False self._connection_attempted = False self._last_connection_check = 0 @@ -142,7 +106,14 @@ class PlexClient: self.server = None def get_available_music_libraries(self) -> List[Dict[str, str]]: - """Get list of all available music libraries on the Plex server""" + """Get list of all available music libraries on the Plex server. + + Prepends an "All Libraries (combined)" synthetic entry whose + ``key`` is ``ALL_LIBRARIES_SENTINEL``. The settings UI renders + this as a normal dropdown option; selecting it stores the + sentinel as the saved preference, which flips the client into + all-libraries read mode (see ``_all_libraries_mode``). + """ if not self.ensure_connection() or not self.server: return [] @@ -152,24 +123,56 @@ class PlexClient: if section.type == 'artist': music_libraries.append({ 'title': section.title, - 'key': str(section.key) + 'key': str(section.key), + # ``value`` is what the frontend submits to + # ``/api/plex/select-music-library``. Real + # libraries use their title (preserves prior + # behavior). The synthetic sentinel entry below + # uses the sentinel string so the backend's + # ``set_music_library_by_name`` can recognize it. + 'value': section.title, }) - logger.debug(f"Found {len(music_libraries)} music libraries") + # Only offer the "All Libraries" option when the user actually + # has more than one music library on their server — single- + # library users don't need the extra menu item. + if len(music_libraries) > 1: + music_libraries.insert(0, { + 'title': 'All Libraries (combined)', + 'key': ALL_LIBRARIES_SENTINEL, + 'value': ALL_LIBRARIES_SENTINEL, + }) + + logger.debug(f"Found {len(music_libraries)} music libraries (incl. sentinel)") return music_libraries except Exception as e: logger.error(f"Error getting music libraries: {e}") return [] def set_music_library_by_name(self, library_name: str) -> bool: - """Set the active music library by name""" + """Set the active music library by name. + + Special-case ``ALL_LIBRARIES_SENTINEL`` (``'__all_libraries__'``): + flips the client into all-libraries mode where read methods union + across every music section instead of querying a single one. + """ if not self.server: return False try: + if library_name == ALL_LIBRARIES_SENTINEL: + self.music_library = None + self._all_libraries_mode = True + logger.info("Set music library to: All Libraries (combined)") + from database.music_database import MusicDatabase + db = MusicDatabase() + db.set_preference('plex_music_library', ALL_LIBRARIES_SENTINEL) + return True + for section in self.server.library.sections(): if section.type == 'artist' and section.title == library_name: self.music_library = section + self._all_libraries_mode = False logger.info(f"Set music library to: {library_name}") # Store preference in database @@ -207,11 +210,23 @@ class PlexClient: db = MusicDatabase() preferred_library = db.get_preference('plex_music_library') + if preferred_library == ALL_LIBRARIES_SENTINEL: + # User opted into all-libraries mode — read methods + # union across every music section. + self.music_library = None + self._all_libraries_mode = True + logger.debug( + f"Using all-libraries mode across " + f"{len(music_sections)} music sections" + ) + return + if preferred_library: # Try to find the preferred library for section in music_sections: if section.title == preferred_library: self.music_library = section + self._all_libraries_mode = False logger.debug(f"Using user-selected music library: {section.title}") return except Exception as e: @@ -265,10 +280,167 @@ class PlexClient: return self.server is not None def is_fully_configured(self) -> bool: - """Check if both server is connected AND music library is selected.""" - return self.server is not None and self.music_library is not None + """Check if both server is connected AND music library is selected + (or user has opted into all-libraries mode).""" + return self.server is not None and ( + self.music_library is not None or self._all_libraries_mode + ) + + def is_all_libraries_mode(self) -> bool: + """True when the user has selected the synthetic "All Libraries + (combined)" option — read methods union across every music + section instead of querying a single one. Public accessor so + external callers don't reach into the underscore-prefixed + flag directly.""" + return self._all_libraries_mode + + # ------------------------------------------------------------------ + # Library scope helpers — single dispatch point for "single section + # vs all music sections" so every read method funnels through the + # same branch. + # ------------------------------------------------------------------ + + def _can_query(self) -> bool: + """True when there's something to query — a selected section or + all-libraries mode + a connected server.""" + if not self.server: + return False + if self._all_libraries_mode: + return True + return self.music_library is not None + + def _get_music_sections(self) -> List: + """Music sections currently in scope (one for single-library + mode, every music section for all-libraries mode). Used by + ``trigger_library_scan`` / ``is_library_scanning`` which take + per-section action.""" + if not self.server: + return [] + if self._all_libraries_mode: + try: + return [s for s in self.server.library.sections() if s.type == 'artist'] + except Exception as exc: + logger.error(f"Error enumerating music sections: {exc}") + return [] + if self.music_library: + return [self.music_library] + return [] + + def _all_artists(self) -> List[PlexArtist]: + """Every artist in the configured scope.""" + if self._all_libraries_mode: + return self.server.library.search(libtype='artist') + if self.music_library: + return self.music_library.searchArtists() + return [] + + def _all_albums(self) -> List[PlexAlbum]: + """Every album in the configured scope.""" + if self._all_libraries_mode: + return self.server.library.search(libtype='album') + if self.music_library: + return self.music_library.albums() + return [] + + def _all_tracks(self) -> List[PlexTrack]: + """Every track in the configured scope.""" + if self._all_libraries_mode: + return self.server.library.search(libtype='track') + if self.music_library: + return self.music_library.searchTracks() + return [] + + def _search_general(self, **kwargs): + """Generic search dispatch — single section or server-wide. + + Used by ``_find_track`` / ``search_tracks`` for fielded + searches (title=, artist=, libtype=, limit=, etc). + """ + if self._all_libraries_mode: + return self.server.library.search(**kwargs) + if self.music_library: + return self.music_library.search(**kwargs) + return [] + + def _search_artists_by_name(self, title: str, limit: int = 1) -> List[PlexArtist]: + """Find artist objects matching a name. Used by ``search_tracks`` + Stage 1 + ``search_albums`` artist-then-filter flow.""" + if self._all_libraries_mode: + results = self.server.library.search(title=title, libtype='artist', limit=limit) + return [r for r in results if isinstance(r, PlexArtist)] + if self.music_library: + return self.music_library.searchArtists(title=title, limit=limit) + return [] + + # ------------------------------------------------------------------ + # Cross-section dedup. Applied ONLY in all-libraries mode and ONLY + # at the listing/stats layer. Never apply to ratingKey enumeration + # (``get_all_artist_ids`` / ``get_all_album_ids``) — removal + # detection compares those sets against DB-linked ratingKeys and + # deduping would falsely prune library tracks pointing at the + # non-canonical ratingKey. Single-library mode is a no-op fast + # path so the dedup code path never touches it. + # ------------------------------------------------------------------ + + def _dedupe_artists(self, artists: List[PlexArtist]) -> List[PlexArtist]: + """Collapse same-name artists across sections to a single + canonical entry (the one with the higher track count). Returns + the input unchanged in single-library mode and when there's + nothing to dedup.""" + if not self._all_libraries_mode or not artists: + return artists + by_name: Dict[str, PlexArtist] = {} + for a in artists: + name_key = (getattr(a, 'title', '') or '').strip().lower() + if not name_key: + # Untitled artist — keep as-is, can't dedup blindly. + by_name[f'__nokey_{getattr(a, "ratingKey", id(a))}'] = a + continue + existing = by_name.get(name_key) + if existing is None: + by_name[name_key] = a + continue + try: + existing_count = getattr(existing, 'leafCount', 0) or 0 + new_count = getattr(a, 'leafCount', 0) or 0 + if new_count > existing_count: + by_name[name_key] = a + except Exception as e: + logger.debug("artist leafCount compare failed: %s", e) + return list(by_name.values()) + + def _dedupe_albums(self, albums: List[PlexAlbum]) -> List[PlexAlbum]: + """Collapse same-album-by-same-artist across sections to a + single canonical entry. Group key is (artist_lower, title_lower); + canonical = higher track count. No-op in single-library mode.""" + if not self._all_libraries_mode or not albums: + return albums + by_key: Dict[tuple, PlexAlbum] = {} + for alb in albums: + title = (getattr(alb, 'title', '') or '').strip().lower() + if not title: + by_key[('__nokey__', getattr(alb, 'ratingKey', id(alb)))] = alb + continue + artist = '' + try: + artist = (getattr(alb, 'parentTitle', '') or '').strip().lower() + except Exception as e: + logger.debug("album parentTitle read failed: %s", e) + key = (artist, title) + existing = by_key.get(key) + if existing is None: + by_key[key] = alb + continue + try: + existing_count = getattr(existing, 'leafCount', 0) or 0 + new_count = getattr(alb, 'leafCount', 0) or 0 + if new_count > existing_count: + by_key[key] = alb + except Exception as e: + logger.debug("album leafCount compare failed: %s", e) + return list(by_key.values()) - def get_all_playlists(self) -> List[PlexPlaylistInfo]: + def get_all_playlists(self) -> List[PlaylistInfo]: if not self.ensure_connection(): logger.error("Not connected to Plex server") return [] @@ -278,7 +450,7 @@ class PlexClient: try: for playlist in self.server.playlists(): if getattr(playlist, 'playlistType', None) == 'audio': - playlist_info = PlexPlaylistInfo.from_plex_playlist(playlist) + playlist_info = PlaylistInfo.from_plex_playlist(playlist) playlists.append(playlist_info) logger.info(f"Retrieved {len(playlists)} audio playlists") @@ -288,14 +460,14 @@ class PlexClient: logger.error(f"Error fetching playlists: {e}") return [] - def get_playlist_by_name(self, name: str) -> Optional[PlexPlaylistInfo]: + def get_playlist_by_name(self, name: str) -> Optional[PlaylistInfo]: if not self.ensure_connection(): return None try: playlist = self.server.playlist(name) if getattr(playlist, 'playlistType', None) == 'audio': - return PlexPlaylistInfo.from_plex_playlist(playlist) + return PlaylistInfo.from_plex_playlist(playlist) return None except NotFound: @@ -311,14 +483,14 @@ class PlexClient: return False try: - # Handle both PlexTrackInfo objects and actual Plex track objects + # Handle both TrackInfo objects and actual Plex track objects plex_tracks = [] for track in tracks: if hasattr(track, 'ratingKey'): # This is already a Plex track object plex_tracks.append(track) elif hasattr(track, '_original_plex_track'): - # This is a PlexTrackInfo object with stored original track reference + # This is a TrackInfo object with stored original track reference original_track = track._original_plex_track if original_track is not None: plex_tracks.append(original_track) @@ -326,7 +498,7 @@ class PlexClient: else: logger.warning(f"Stored track reference is None for: {track.title} by {track.artist}") elif hasattr(track, 'title'): - # Fallback: This is a PlexTrackInfo object, need to find the actual track + # Fallback: This is a TrackInfo object, need to find the actual track plex_track = self._find_track(track.title, track.artist, track.album) if plex_track: plex_tracks.append(plex_track) @@ -448,7 +620,7 @@ class PlexClient: logger.error(f"Error copying playlist '{source_name}' to '{target_name}': {e}") return False - def update_playlist(self, playlist_name: str, tracks: List[PlexTrackInfo]) -> bool: + def update_playlist(self, playlist_name: str, tracks: List[TrackInfo]) -> bool: if not self.ensure_connection(): return False @@ -493,38 +665,38 @@ class PlexClient: return False def _find_track(self, title: str, artist: str, album: str) -> Optional[PlexTrack]: - if not self.music_library: + if not self._can_query(): return None - + try: - search_results = self.music_library.search(title=title, artist=artist, album=album) - + search_results = self._search_general(title=title, artist=artist, album=album) + for result in search_results: if isinstance(result, PlexTrack): - if (result.title.lower() == title.lower() and + if (result.title.lower() == title.lower() and result.artist().title.lower() == artist.lower() and result.album().title.lower() == album.lower()): return result - - broader_search = self.music_library.search(title=title, artist=artist) + + broader_search = self._search_general(title=title, artist=artist) for result in broader_search: if isinstance(result, PlexTrack): - if (result.title.lower() == title.lower() and + if (result.title.lower() == title.lower() and result.artist().title.lower() == artist.lower()): return result - + return None - + except Exception as e: logger.error(f"Error searching for track '{title}' by '{artist}': {e}") return None - def search_tracks(self, title: str, artist: str, limit: int = 15) -> List[PlexTrackInfo]: + def search_tracks(self, title: str, artist: str, limit: int = 15) -> List[TrackInfo]: """ Searches for tracks using an efficient, multi-stage "early exit" strategy. It stops and returns results as soon as candidates are found. """ - if not self.music_library: + if not self._can_query(): logger.warning("Plex music library not found. Cannot perform search.") return [] @@ -542,7 +714,7 @@ class PlexClient: # --- Stage 1: High-Precision Search (Artist -> then filter by Title) --- if artist: logger.debug(f"Stage 1: Searching for artist '{artist}'") - artist_results = self.music_library.searchArtists(title=artist, limit=1) + artist_results = self._search_artists_by_name(title=artist, limit=1) if artist_results: plex_artist = artist_results[0] all_artist_tracks = plex_artist.tracks() @@ -554,7 +726,7 @@ class PlexClient: # --- Early Exit: If Stage 1 found results, stop here --- if candidate_tracks: logger.info(f"Found {len(candidate_tracks)} candidates in Stage 1. Exiting early.") - tracks = [PlexTrackInfo.from_plex_track(track) for track in candidate_tracks[:limit]] + tracks = [TrackInfo.from_plex_track(track) for track in candidate_tracks[:limit]] # Store references to original tracks for playlist creation for i, track_info in enumerate(tracks): if i < len(candidate_tracks): @@ -567,13 +739,13 @@ class PlexClient: # --- Stage 2: Flexible Keyword Search (Artist + Title combined) --- search_query = f"{artist} {title}".strip() logger.debug(f"Stage 2: Performing keyword search for '{search_query}'") - stage2_results = self.music_library.search(title=search_query, libtype='track', limit=limit) + stage2_results = self._search_general(title=search_query, libtype='track', limit=limit) add_candidates(stage2_results) # --- Early Exit: If Stage 2 found results, stop here --- if candidate_tracks: logger.info(f"Found {len(candidate_tracks)} candidates in Stage 2. Exiting early.") - tracks = [PlexTrackInfo.from_plex_track(track) for track in candidate_tracks[:limit]] + tracks = [TrackInfo.from_plex_track(track) for track in candidate_tracks[:limit]] # Store references to original tracks for playlist creation for i, track_info in enumerate(tracks): if i < len(candidate_tracks): @@ -587,7 +759,7 @@ class PlexClient: # Removed to prevent false positives where tracks with same title # but different artists are incorrectly matched - tracks = [PlexTrackInfo.from_plex_track(track) for track in candidate_tracks[:limit]] + tracks = [TrackInfo.from_plex_track(track) for track in candidate_tracks[:limit]] # Store references to original tracks for playlist creation for i, track_info in enumerate(tracks): @@ -612,18 +784,106 @@ class PlexClient: def get_library_stats(self) -> Dict[str, int]: - if not self.music_library: + if not self._can_query(): return {} - + try: + # Apply dedup to artist + album counts so stats match what + # the user actually sees in the library list (deduped). Tracks + # stay raw — same track in two sections means two distinct + # files / Plex entries, not a logical duplicate. return { - 'artists': len(self.music_library.searchArtists()), - 'albums': len(self.music_library.searchAlbums()), - 'tracks': len(self.music_library.searchTracks()) + 'artists': len(self._dedupe_artists(self._all_artists())), + 'albums': len(self._dedupe_albums(self._all_albums())), + 'tracks': len(self._all_tracks()) } except Exception as e: logger.error(f"Error getting library stats: {e}") return {} + + def get_recently_added_albums(self, maxresults: int = 400, + libtype: Optional[str] = 'album') -> List: + """Recently added items across the configured scope. + + In all-libraries mode, iterates every music section and concats + each section's ``recentlyAdded`` list. Honors ``maxresults`` + per-section to bound the response. Caller is responsible for + further sorting / deduping if needed. + + Pass ``libtype=None`` to skip the type filter (returns mixed + artists / albums / tracks). Used by the database update worker's + deep-scan recent-content sweep — pre-fix it reached + ``self.music_library.recentlyAdded()`` directly which crashed + in all-libraries mode (music_library is None there). + """ + if not self.ensure_connection() or not self._can_query(): + return [] + + def _call(target): + try: + if libtype is not None: + return target.recentlyAdded(libtype=libtype, maxresults=maxresults) + return target.recentlyAdded(maxresults=maxresults) + except TypeError: + # Older PlexAPI versions don't accept libtype/maxresults + # as kwargs — fall back to bare call. + return target.recentlyAdded() + + try: + if self._all_libraries_mode: + aggregate = [] + for section in self._get_music_sections(): + try: + aggregate.extend(_call(section)) + except Exception as inner_exc: + logger.debug(f"recentlyAdded failed for section '{section.title}': {inner_exc}") + return aggregate + if self.music_library: + return _call(self.music_library) + return [] + except Exception as exc: + logger.error(f"Error getting recently added albums: {exc}") + return [] + + def get_recently_updated_albums(self, limit: int = 400) -> List: + """Albums sorted by ``updatedAt`` descending across the scope. + + Used by the deep-scan to catch metadata corrections (renames, + re-tagging) that recently-added doesn't surface. In all- + libraries mode, unions across every music section. + """ + if not self.ensure_connection() or not self._can_query(): + return [] + try: + return self._search_general(sort='updatedAt:desc', libtype='album', limit=limit) + except Exception as exc: + logger.error(f"Error getting recently updated albums: {exc}") + return [] + + def get_music_library_locations(self) -> List[str]: + """Folder roots configured for the music library scope. + + Single-library mode returns the selected section's locations. + All-libraries mode unions locations across every music section + — needed by ``web_server.py``'s file-path resolver to recognize + files under any music root, not just one. + """ + if not self.ensure_connection() or not self._can_query(): + return [] + try: + sections = self._get_music_sections() + locations = [] + for section in sections: + try: + for loc in section.locations: + if loc and loc not in locations: + locations.append(loc) + except Exception as inner_exc: + logger.debug(f"Could not read locations for section '{getattr(section, 'title', '?')}': {inner_exc}") + return locations + except Exception as exc: + logger.error(f"Error getting music library locations: {exc}") + return [] def get_play_history(self, limit=500): """Fetch recently played tracks from Plex. @@ -663,11 +923,11 @@ class PlexClient: Returns dict of {ratingKey: play_count}. """ - if not self.ensure_connection() or not self.music_library: + if not self.ensure_connection() or not self._can_query(): return {} try: - tracks = self.music_library.searchTracks() + tracks = self._all_tracks() counts = {} for track in tracks: view_count = getattr(track, 'viewCount', 0) or 0 @@ -680,14 +940,27 @@ class PlexClient: return {} def get_all_artists(self) -> List[PlexArtist]: - """Get all artists from the music library""" - if not self.ensure_connection() or not self.music_library: + """Get all artists from the music library. + + In all-libraries mode, dedupes same-name artists across + sections (canonical = higher track count) so the library list + doesn't show "Drake" twice when Drake is in two sections. + Single-library mode is unaffected — dedup helper is a no-op. + """ + if not self.ensure_connection() or not self._can_query(): logger.error("Not connected to Plex server or no music library") return [] try: - artists = self.music_library.searchArtists() - logger.info(f"Found {len(artists)} artists in Plex library") + raw = self._all_artists() + artists = self._dedupe_artists(raw) + if len(raw) != len(artists): + logger.info( + f"Found {len(raw)} artists across all music sections " + f"({len(artists)} unique after cross-section dedup)" + ) + else: + logger.info(f"Found {len(artists)} artists in Plex library") return artists except Exception as e: logger.error(f"Error getting all artists: {e}") @@ -695,10 +968,10 @@ class PlexClient: def get_all_artist_ids(self) -> set: """Get all artist IDs from Plex library (lightweight, for removal detection).""" - if not self.ensure_connection() or not self.music_library: + if not self.ensure_connection() or not self._can_query(): return set() try: - artists = self.music_library.searchArtists() + artists = self._all_artists() ids = {str(a.ratingKey) for a in artists} logger.info(f"Retrieved {len(ids)} artist IDs from Plex") return ids @@ -708,10 +981,10 @@ class PlexClient: def get_all_album_ids(self) -> set: """Get all album IDs from Plex library (lightweight, for removal detection).""" - if not self.ensure_connection() or not self.music_library: + if not self.ensure_connection() or not self._can_query(): return set() try: - albums = self.music_library.albums() + albums = self._all_albums() ids = {str(a.ratingKey) for a in albums} logger.info(f"Retrieved {len(ids)} album IDs from Plex") return ids @@ -921,11 +1194,31 @@ class PlexClient: return False def trigger_library_scan(self, library_name: str = "Music") -> bool: - """Trigger Plex library scan for the specified library""" + """Trigger Plex library scan. + + In all-libraries mode, fans the scan trigger across every music + section. Otherwise targets the named section (default ``Music``). + Returns True if at least one section was successfully triggered. + """ if not self.ensure_connection(): return False - + try: + if self._all_libraries_mode: + sections = self._get_music_sections() + if not sections: + logger.warning("All-libraries mode active but no music sections found") + return False + triggered = 0 + for section in sections: + try: + section.update() + triggered += 1 + except Exception as inner_exc: + logger.error(f"Failed to trigger scan for '{section.title}': {inner_exc}") + logger.info(f"Triggered Plex library scan across {triggered}/{len(sections)} music sections") + return triggered > 0 + library = self.server.library.section(library_name) library.update() # Non-blocking scan request logger.info(f"Triggered Plex library scan for '{library_name}'") @@ -933,66 +1226,93 @@ class PlexClient: except Exception as e: logger.error(f"Failed to trigger library scan for '{library_name}': {e}") return False - + def is_library_scanning(self, library_name: str = "Music") -> bool: - """Check if Plex library is currently scanning""" + """Check if Plex library is currently scanning. + + In all-libraries mode, returns True if ANY music section is + scanning. Otherwise checks the named section. + """ if not self.ensure_connection(): logger.debug("Not connected to Plex, cannot check scan status") return False - + try: + if self._all_libraries_mode: + sections = self._get_music_sections() + # Per-section refreshing flag check. + for section in sections: + if hasattr(section, 'refreshing') and section.refreshing: + logger.debug(f"Section '{section.title}' is refreshing") + return True + # Activity feed check — match any music-section title. + try: + activities = self.server.activities() + section_titles = {s.title.lower() for s in sections} + for activity in activities: + activity_type = getattr(activity, 'type', 'unknown') + activity_title = getattr(activity, 'title', '') or '' + if activity_type not in ['library.scan', 'library.refresh']: + continue + if any(title in activity_title.lower() for title in section_titles): + logger.debug(f"Matching scan activity: {activity_title}") + return True + except Exception as activities_error: + logger.debug(f"Could not check server activities: {activities_error}") + return False + library = self.server.library.section(library_name) - + # Check if library has a scanning attribute or is refreshing # The Plex API exposes this through the library's refreshing property refreshing = hasattr(library, 'refreshing') and library.refreshing logger.debug("Library.refreshing = %s", refreshing) - + if refreshing: logger.debug("Library is refreshing") return True - + # Alternative method: Check server activities for scanning try: activities = self.server.activities() logger.debug("Found %s server activities", len(activities)) - + for activity in activities: # Look for library scan activities activity_type = getattr(activity, 'type', 'unknown') activity_title = getattr(activity, 'title', 'unknown') logger.debug("Activity - type=%s title=%s", activity_type, activity_title) - + if (activity_type in ['library.scan', 'library.refresh'] and library_name.lower() in activity_title.lower()): logger.debug("Found matching scan activity: %s", activity_title) return True except Exception as activities_error: logger.debug(f"Could not check server activities: {activities_error}") - + logger.debug("No scan activity detected") return False - + except Exception as e: logger.debug(f"Error checking if library is scanning: {e}") return False def search_albums(self, album_name: str = "", artist_name: str = "", limit: int = 20) -> List[Dict[str, Any]]: """Search for albums in Plex library""" - if not self.ensure_connection() or not self.music_library: + if not self.ensure_connection() or not self._can_query(): return [] - + try: albums = [] - + # Perform search - different approaches based on what we're searching for search_results = [] - + if album_name and artist_name: # Search for albums by specific artist and title try: # First try searching for the artist, then filter their albums - artist_results = self.music_library.searchArtists(title=artist_name, limit=3) + artist_results = self._search_artists_by_name(title=artist_name, limit=3) for artist in artist_results: try: artist_albums = artist.albums() @@ -1005,25 +1325,25 @@ class PlexClient: logger.debug(f"Artist search failed, trying general search: {e}") # Fallback to general album search try: - search_results = self.music_library.search(title=album_name) + search_results = self._search_general(title=album_name) # Filter to only albums search_results = [r for r in search_results if isinstance(r, PlexAlbum)] except Exception as e2: logger.debug(f"General search also failed: {e2}") - + elif album_name: # Search for albums by title only try: - search_results = self.music_library.search(title=album_name) - # Filter to only albums + search_results = self._search_general(title=album_name) + # Filter to only albums search_results = [r for r in search_results if isinstance(r, PlexAlbum)] except Exception as e: logger.debug(f"Album title search failed: {e}") - + elif artist_name: # Search for all albums by artist try: - artist_results = self.music_library.searchArtists(title=artist_name, limit=1) + artist_results = self._search_artists_by_name(title=artist_name, limit=1) if artist_results: search_results = artist_results[0].albums() except Exception as e: @@ -1031,7 +1351,7 @@ class PlexClient: else: # Get all albums if no search terms try: - search_results = self.music_library.albums() + search_results = self._all_albums() except Exception as e: logger.debug(f"Get all albums failed: {e}") diff --git a/core/qobuz_client.py b/core/qobuz_client.py index e7dcad9d..1cb58a66 100644 --- a/core/qobuz_client.py +++ b/core/qobuz_client.py @@ -28,7 +28,7 @@ from utils.logging_config import get_logger from config.settings import config_manager # Import Soulseek data structures for drop-in replacement compatibility -from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus +from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus logger = get_logger("qobuz_client") @@ -104,7 +104,10 @@ QOBUZ_QUALITY_MAP = { } -class QobuzClient: +from core.download_plugins.base import DownloadSourcePlugin + + +class QobuzClient(DownloadSourcePlugin): """ Qobuz download client using Qobuz REST API. Provides search, matching, and download capabilities as a drop-in alternative to Soulseek/YouTube/Tidal. @@ -136,13 +139,19 @@ class QobuzClient: self.user_info: Optional[Dict] = None self._auth_error: Optional[str] = None - # Download queue management - self.active_downloads: Dict[str, Dict[str, Any]] = {} - self._download_lock = threading.Lock() + # Engine reference is populated by set_engine() at registration + # time. None until orchestrator wires the registry. + self._engine = None # Try to restore saved session self._restore_session() + def set_engine(self, engine): + """Engine callback — gives the client access to the central + thread worker + state store. Engine calls this during + ``register_plugin`` if the plugin defines it.""" + self._engine = engine + def set_shutdown_check(self, check_callable): """Set a callback function to check for system shutdown""" self.shutdown_check = check_callable @@ -516,6 +525,45 @@ class QobuzClient: config_manager.set('qobuz.session', {}) logger.info("Qobuz session cleared") + def reload_credentials(self) -> None: + """Pull session state from config without making a network probe. + + SoulSync runs two ``QobuzClient`` instances side by side — one wired + through ``download_orchestrator.client('qobuz')`` for the auth-flow endpoints, and a + second owned by the enrichment worker for thread safety. When the user + logs in via ``/api/qobuz/auth/login`` or ``/api/qobuz/auth/token`` only + the auth-flow instance's in-memory state is updated; the worker's + instance still believes itself unauthenticated, which is what made the + dashboard "yellow" indicator and the connection-test step report + ``Qobuz not authenticated`` even after a successful Connect. + + Call this on the worker's client immediately after a successful login + (and on logout, to clear) to keep the two instances in lockstep. + Unlike ``_restore_session`` this does not validate the token over the + network — the caller has just authenticated, so the token is known + good. + """ + saved = config_manager.get('qobuz.session', {}) or {} + new_app_id = saved.get('app_id', '') or None + new_app_secret = saved.get('app_secret', '') or None + new_token = saved.get('user_auth_token', '') or None + + self.app_id = new_app_id + self.app_secret = new_app_secret + self.user_auth_token = new_token + + if new_app_id: + self.session.headers['X-App-Id'] = new_app_id + else: + self.session.headers.pop('X-App-Id', None) + + if new_token: + self.session.headers['X-User-Auth-Token'] = new_token + else: + self.session.headers.pop('X-User-Auth-Token', None) + self.user_info = None + self._auth_error = None + def is_authenticated(self) -> bool: """Check if we have a valid Qobuz session.""" return bool(self.user_auth_token and self.app_id and self.app_secret) @@ -845,97 +893,38 @@ class QobuzClient: return None async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]: - """ - Download a Qobuz track (async, Soulseek-compatible interface). + """Download a Qobuz track. Delegates to engine.worker which + spawns the background thread + manages state.""" + if '||' not in filename: + logger.error(f"Invalid filename format: {filename}") + return None + if self._engine is None: + # Raise rather than return None so the orchestrator's + # download_with_fallback surfaces a real warning + tries + # the next source. Returning None silently dropped the + # download with no user feedback (per JohnBaumb). + raise RuntimeError("Qobuz client has no engine reference — cannot dispatch download") - Returns download_id immediately and runs download in background thread. - - Args: - username: Ignored for Qobuz (always "qobuz") - filename: Encoded as "track_id||display_name" - file_size: Ignored - """ + track_id_str, display_name = filename.split('||', 1) try: - if '||' not in filename: - logger.error(f"Invalid filename format: {filename}") - return None - - track_id_str, display_name = filename.split('||', 1) - try: - track_id = int(track_id_str) - except ValueError: - logger.error(f"Invalid Qobuz track ID: {track_id_str}") - return None - - logger.info(f"Starting Qobuz download: {display_name}") - - download_id = str(uuid.uuid4()) - - with self._download_lock: - self.active_downloads[download_id] = { - 'id': download_id, - 'filename': filename, - 'username': 'qobuz', - 'state': 'Initializing', - 'progress': 0.0, - 'size': 0, - 'transferred': 0, - 'speed': 0, - 'time_remaining': None, - 'track_id': track_id, - 'display_name': display_name, - 'file_path': None, - } - - # Start download in background thread - download_thread = threading.Thread( - target=self._download_thread_worker, - args=(download_id, track_id, display_name, filename), - daemon=True, - ) - download_thread.start() - - logger.info(f"Qobuz download {download_id} started in background") - return download_id - - except Exception as e: - logger.error(f"Failed to start Qobuz download: {e}") - import traceback - traceback.print_exc() + track_id = int(track_id_str) + except ValueError: + logger.error(f"Invalid Qobuz track ID: {track_id_str}") return None - def _download_thread_worker(self, download_id: str, track_id: int, display_name: str, original_filename: str): - """Background thread worker for downloading Qobuz tracks.""" - try: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'InProgress, Downloading' + logger.info(f"Starting Qobuz download: {display_name}") - file_path = self._download_sync(download_id, track_id, display_name) - - if file_path: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Completed, Succeeded' - self.active_downloads[download_id]['progress'] = 100.0 - self.active_downloads[download_id]['file_path'] = file_path - - logger.info(f"Qobuz download {download_id} completed: {file_path}") - else: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' - - logger.error(f"Qobuz download {download_id} failed") - - except Exception as e: - logger.error(f"Qobuz download thread failed for {download_id}: {e}") - import traceback - traceback.print_exc() - - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' + return self._engine.worker.dispatch( + source_name='qobuz', + target_id=track_id, + display_name=display_name, + original_filename=filename, + impl_callable=self._download_sync, + extra_record_fields={ + 'track_id': track_id, + 'display_name': display_name, + }, + ) def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]: """ @@ -1011,9 +1000,8 @@ class QobuzClient: chunk_size = 64 * 1024 # 64KB chunks start_time = time.time() - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['size'] = total_size + if self._engine is not None: + self._engine.update_record('qobuz', download_id, {'size': total_size}) with open(out_path, 'wb') as f: for chunk in response.iter_content(chunk_size=chunk_size): @@ -1022,9 +1010,10 @@ class QobuzClient: # Check for shutdown or cancellation cancelled = False - with self._download_lock: - if download_id in self.active_downloads: - cancelled = self.active_downloads[download_id].get('state') == 'Cancelled' + if self._engine is not None: + rec = self._engine.get_record('qobuz', download_id) + if rec is not None: + cancelled = rec.get('state') == 'Cancelled' if cancelled or (self.shutdown_check and self.shutdown_check()): reason = "cancelled" if cancelled else "server shutting down" logger.info(f"Aborting Qobuz download mid-stream: {reason}") @@ -1045,18 +1034,20 @@ class QobuzClient: progress = 0 time_remaining = None - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['transferred'] = downloaded - self.active_downloads[download_id]['progress'] = round(progress, 1) - self.active_downloads[download_id]['speed'] = int(speed) - self.active_downloads[download_id]['time_remaining'] = time_remaining + if self._engine is not None: + self._engine.update_record('qobuz', download_id, { + 'transferred': downloaded, + 'progress': round(progress, 1), + 'speed': int(speed), + 'time_remaining': time_remaining, + }) # If download was aborted (shutdown/cancel), clean up partial file abort_check = False - with self._download_lock: - if download_id in self.active_downloads: - abort_check = self.active_downloads[download_id].get('state') == 'Cancelled' + if self._engine is not None: + rec = self._engine.get_record('qobuz', download_id) + if rec is not None: + abort_check = rec.get('state') == 'Cancelled' if abort_check or (self.shutdown_check and self.shutdown_check()): out_path.unlink(missing_ok=True) return None @@ -1100,79 +1091,55 @@ class QobuzClient: # ===================== Status / Cancel / Clear ===================== + def _record_to_status(self, record): + return DownloadStatus( + id=record['id'], + filename=record['filename'], + username=record['username'], + state=record['state'], + progress=record['progress'], + size=record.get('size', 0), + transferred=record.get('transferred', 0), + speed=record.get('speed', 0), + time_remaining=record.get('time_remaining'), + file_path=record.get('file_path'), + ) + async def get_all_downloads(self) -> List[DownloadStatus]: - """Get all active downloads (matches Soulseek interface).""" - download_statuses = [] - - with self._download_lock: - for _download_id, info in self.active_downloads.items(): - status = DownloadStatus( - id=info['id'], - filename=info['filename'], - username=info['username'], - state=info['state'], - progress=info['progress'], - size=info['size'], - transferred=info['transferred'], - speed=info['speed'], - time_remaining=info.get('time_remaining'), - file_path=info.get('file_path'), - ) - download_statuses.append(status) - - return download_statuses + if self._engine is None: + return [] + return [ + self._record_to_status(record) + for record in self._engine.iter_records_for_source('qobuz') + ] async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: - """Get status of a specific download (matches Soulseek interface).""" - with self._download_lock: - if download_id not in self.active_downloads: - return None - - info = self.active_downloads[download_id] - return DownloadStatus( - id=info['id'], - filename=info['filename'], - username=info['username'], - state=info['state'], - progress=info['progress'], - size=info['size'], - transferred=info['transferred'], - speed=info['speed'], - time_remaining=info.get('time_remaining'), - file_path=info.get('file_path'), - ) + if self._engine is None: + return None + record = self._engine.get_record('qobuz', download_id) + return self._record_to_status(record) if record is not None else None async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool: - """Cancel an active download (matches Soulseek interface).""" - try: - with self._download_lock: - if download_id not in self.active_downloads: - logger.warning(f"Download {download_id} not found") - return False - - self.active_downloads[download_id]['state'] = 'Cancelled' - logger.info(f"Marked Qobuz download {download_id} as cancelled") - - if remove: - del self.active_downloads[download_id] - logger.info(f"Removed Qobuz download {download_id} from queue") - - return True - except Exception as e: - logger.error(f"Failed to cancel download {download_id}: {e}") + if self._engine is None: return False + if self._engine.get_record('qobuz', download_id) is None: + logger.warning(f"Qobuz download {download_id} not found") + return False + self._engine.update_record('qobuz', download_id, {'state': 'Cancelled'}) + logger.info(f"Marked Qobuz download {download_id} as cancelled") + if remove: + self._engine.remove_record('qobuz', download_id) + logger.info(f"Removed Qobuz download {download_id} from queue") + return True async def clear_all_completed_downloads(self) -> bool: - """Clear all terminal downloads from the list (matches Soulseek interface).""" + if self._engine is None: + return True try: - with self._download_lock: - ids_to_remove = [ - did for did, info in self.active_downloads.items() - if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted') - ] - for did in ids_to_remove: - del self.active_downloads[did] - + terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} + for record in list(self._engine.iter_records_for_source('qobuz')): + if record.get('state') in terminal: + self._engine.remove_record('qobuz', record['id']) return True except Exception as e: logger.error(f"Error clearing downloads: {e}") diff --git a/core/qobuz_worker.py b/core/qobuz_worker.py index 5e85c508..c6ba2be7 100644 --- a/core/qobuz_worker.py +++ b/core/qobuz_worker.py @@ -9,6 +9,7 @@ from utils.logging_config import get_logger from database.music_database import MusicDatabase from core.qobuz_client import _qobuz_is_rate_limited from core.worker_utils import interruptible_sleep +from core.enrichment.manual_match_honoring import honor_stored_match logger = get_logger("qobuz_worker") @@ -103,8 +104,8 @@ class QobuzWorker: try: if self.client: authenticated = self.client.is_authenticated() - except Exception: - pass + except Exception as e: + logger.debug("qobuz auth status check: %s", e) return { 'enabled': True, @@ -162,8 +163,8 @@ class QobuzWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null-id item type lookup: %s", e) continue @@ -426,12 +427,26 @@ class QobuzWorker: self.stats['not_found'] += 1 logger.debug(f"No match for artist '{artist_name}'") + def _refresh_album_via_stored_id(self, album_id, stored_id, full_album_dict): + """Issue #501 callback. Same shape as Tidal/Deezer — pass the + full-album dict in both arg slots.""" + self._update_album(album_id, full_album_dict, full_album_dict) + + def _refresh_track_via_stored_id(self, track_id, stored_id, full_track_dict): + self._update_track(track_id, full_track_dict, full_track_dict) + def _process_album(self, album_id: int, album_name: str, artist_name: str, item: Dict[str, Any]): """Process an album: search Qobuz, verify, fetch full details, store metadata""" - existing_id = self._get_existing_id('album', album_id) - if existing_id: - logger.debug(f"Preserving existing Qobuz ID for album '{album_name}': {existing_id}") - self._mark_status('album', album_id, 'matched') + # Issue #501: honor manual matches. Pre-fix this just marked + # status='matched' without refreshing metadata. + if honor_stored_match( + db=self.db, entity_table='albums', entity_id=album_id, + id_column='qobuz_id', + client_fetch_fn=self.client.get_album, + on_match_fn=self._refresh_album_via_stored_id, + log_prefix='Qobuz', + ): + self.stats['matched'] += 1 return result = self.client.search_album(artist_name, album_name) @@ -484,10 +499,15 @@ class QobuzWorker: def _process_track(self, track_id: int, track_name: str, artist_name: str, item: Dict[str, Any]): """Process a track: search Qobuz, verify, fetch full details, store metadata""" - existing_id = self._get_existing_id('track', track_id) - if existing_id: - logger.debug(f"Preserving existing Qobuz ID for track '{track_name}': {existing_id}") - self._mark_status('track', track_id, 'matched') + # Issue #501: honor manual matches. + if honor_stored_match( + db=self.db, entity_table='tracks', entity_id=track_id, + id_column='qobuz_id', + client_fetch_fn=self.client.get_track, + on_match_fn=self._refresh_track_via_stored_id, + log_prefix='Qobuz', + ): + self.stats['matched'] += 1 return result = self.client.search_track(artist_name, track_name) diff --git a/core/reorganize_runner.py b/core/reorganize_runner.py index b5fba687..8ef7817b 100644 --- a/core/reorganize_runner.py +++ b/core/reorganize_runner.py @@ -95,15 +95,15 @@ def build_runner( def _cleanup_empty(src_dir): try: cleanup_empty_directories_fn(transfer_dir, os.path.join(src_dir, '_')) - except Exception: - pass + except Exception as e: + logger.debug("cleanup empty dirs failed: %s", e) def _on_progress(updates): try: get_queue().update_active_progress(queue_id=item.queue_id, **updates) - except Exception: + except Exception as e: # Progress fan-out failures must never break a run. - pass + logger.debug("reorganize progress fan-out: %s", e) return reorganize_album( album_id=item.album_id, diff --git a/core/repair_jobs/acoustid_scanner.py b/core/repair_jobs/acoustid_scanner.py index 2f27f741..ba3c1564 100644 --- a/core/repair_jobs/acoustid_scanner.py +++ b/core/repair_jobs/acoustid_scanner.py @@ -212,7 +212,7 @@ class AcoustIDScannerJob(RepairJob): ) if context.create_finding: severity = 'warning' if best_score >= 0.90 else 'info' - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='acoustid_mismatch', severity=severity, @@ -240,7 +240,10 @@ class AcoustIDScannerJob(RepairJob): 'track_number': expected.get('track_number'), } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 def _load_db_tracks(self, context: JobContext) -> dict: """Load all tracks from DB keyed by track ID.""" @@ -289,9 +292,14 @@ class AcoustIDScannerJob(RepairJob): return None if os.path.exists(file_path): return file_path - # Try the repair_worker's resolver - from core.repair_worker import _resolve_file_path - return _resolve_file_path(file_path, context.transfer_folder) + # Use the shared library-path resolver — picks up + # library.music_paths and Plex library locations too. + from core.library.path_resolver import resolve_library_file_path + return resolve_library_file_path( + file_path, + transfer_folder=context.transfer_folder, + config_manager=context.config_manager, + ) def _save_checkpoint_id(self, context: JobContext, track_id): """Save or clear the scan checkpoint by track ID.""" diff --git a/core/repair_jobs/album_completeness.py b/core/repair_jobs/album_completeness.py index c92363f3..24b1260f 100644 --- a/core/repair_jobs/album_completeness.py +++ b/core/repair_jobs/album_completeness.py @@ -234,7 +234,7 @@ class AlbumCompletenessJob(RepairJob): ) if context.create_finding: try: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='incomplete_album', severity='info', @@ -264,7 +264,10 @@ class AlbumCompletenessJob(RepairJob): 'artist_thumb_url': artist_thumb or None, } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating completeness finding for album %s: %s", album_id, e) result.errors += 1 diff --git a/core/repair_jobs/album_tag_consistency.py b/core/repair_jobs/album_tag_consistency.py index c6705795..1ae32254 100644 --- a/core/repair_jobs/album_tag_consistency.py +++ b/core/repair_jobs/album_tag_consistency.py @@ -62,8 +62,8 @@ def _read_tag(audio, tag_name): if vals: return vals[0].decode('utf-8') if isinstance(vals[0], bytes) else str(vals[0]) return None - except Exception: - pass + except Exception as e: + logger.debug("read tag value failed: %s", e) return None @@ -288,7 +288,8 @@ class AlbumTagConsistencyJob(RepairJob): variants_str = ' vs '.join(f'"{v}"' for v in inc['variants'][:3]) desc_parts.append(f"{inc['field']}: {variants_str}") - context.create_finding( + inserted = context.create_finding( + job_id=self.job_id, finding_type='album_tag_inconsistency', severity='warning', entity_type='album', @@ -306,7 +307,10 @@ class AlbumTagConsistencyJob(RepairJob): 'file_path': t['file_path']} for t in tag_data], } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 if context.report_progress: context.report_progress( diff --git a/core/repair_jobs/base.py b/core/repair_jobs/base.py index f07b6019..4ac7f52b 100644 --- a/core/repair_jobs/base.py +++ b/core/repair_jobs/base.py @@ -11,6 +11,7 @@ class JobResult: """Result of a single job scan run.""" scanned: int = 0 findings_created: int = 0 + findings_skipped_dedup: int = 0 # Findings the worker already had a row for auto_fixed: int = 0 errors: int = 0 skipped: int = 0 diff --git a/core/repair_jobs/dead_file_cleaner.py b/core/repair_jobs/dead_file_cleaner.py index de87758b..f0b3fb9d 100644 --- a/core/repair_jobs/dead_file_cleaner.py +++ b/core/repair_jobs/dead_file_cleaner.py @@ -2,6 +2,7 @@ import os +from core.library.path_resolver import resolve_library_file_path from core.repair_jobs import register_job from core.repair_jobs.base import JobContext, JobResult, RepairJob from utils.logging_config import get_logger @@ -9,29 +10,14 @@ from utils.logging_config import get_logger logger = get_logger("repair_job.dead_files") -def _resolve_file_path(file_path, transfer_folder, download_folder=None): - """Resolve a stored DB path to an actual file on disk. - - Mirrors _resolve_library_file_path from web_server.py — tries the raw - path first, then progressively shorter suffixes against configured dirs. - """ - if not file_path: - return None - if os.path.exists(file_path): - return file_path - - path_parts = file_path.replace('\\', '/').split('/') - - for base_dir in [transfer_folder, download_folder]: - if not base_dir or not os.path.isdir(base_dir): - continue - # Skip index 0 to avoid drive letter issues (e.g. E:) - for i in range(1, len(path_parts)): - candidate = os.path.join(base_dir, *path_parts[i:]) - if os.path.exists(candidate): - return candidate - - return None +def _resolve_file_path(file_path, transfer_folder, download_folder=None, config_manager=None): + """Backwards-compat wrapper. Use ``resolve_library_file_path`` directly.""" + return resolve_library_file_path( + file_path, + transfer_folder=transfer_folder, + download_folder=download_folder, + config_manager=config_manager, + ) @register_job @@ -122,7 +108,8 @@ class DeadFileCleanerJob(RepairJob): ) # Use the same path resolution logic as library playback - resolved = _resolve_file_path(file_path, context.transfer_folder, download_folder) + resolved = _resolve_file_path(file_path, context.transfer_folder, download_folder, + config_manager=context.config_manager) if resolved is None: # File is truly missing — create finding @@ -133,7 +120,7 @@ class DeadFileCleanerJob(RepairJob): ) if context.create_finding: try: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='dead_file', severity='warning', @@ -152,7 +139,10 @@ class DeadFileCleanerJob(RepairJob): 'artist_thumb_url': artist_thumb or None, } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating dead file finding for track %s: %s", track_id, e) result.errors += 1 diff --git a/core/repair_jobs/discography_backfill.py b/core/repair_jobs/discography_backfill.py index 3170624a..9b7890ca 100644 --- a/core/repair_jobs/discography_backfill.py +++ b/core/repair_jobs/discography_backfill.py @@ -299,8 +299,8 @@ class DiscographyBackfillJob(RepairJob): track_id = track_item.get('id', '') if track_id and self._is_in_wishlist(context.db, track_id): continue - except Exception: - pass + except Exception as e: + logger.debug("wishlist membership check failed: %s", e) # Build wishlist-ready track data. album is a dict (required by # add_to_wishlist and by the download pipeline's cover-art @@ -320,7 +320,7 @@ class DiscographyBackfillJob(RepairJob): # Create finding if context.create_finding: try: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='missing_discography_track', severity='info', @@ -340,13 +340,18 @@ class DiscographyBackfillJob(RepairJob): 'source': source, }, ) - result.findings_created += 1 - missing_count += 1 + if inserted: + result.findings_created += 1 + missing_count += 1 + else: + result.findings_skipped_dedup += 1 # Auto-wishlist mode: also push to wishlist now. The # finding still gets created so the user has a log of - # what the backfill picked up. - if auto_add: + # what the backfill picked up. Only fire on a NEW + # finding — skip if dedup-suppressed (already on the + # wishlist or already auto-added in a prior scan). + if auto_add and inserted: try: context.db.add_to_wishlist( spotify_track_data=track_data, diff --git a/core/repair_jobs/duplicate_detector.py b/core/repair_jobs/duplicate_detector.py index b475e363..3b40f396 100644 --- a/core/repair_jobs/duplicate_detector.py +++ b/core/repair_jobs/duplicate_detector.py @@ -271,7 +271,7 @@ class DuplicateDetectorJob(RepairJob): if context.create_finding: try: group.sort(key=lambda t: (t['bitrate'] or 0), reverse=True) - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='duplicate_tracks', severity='info', @@ -295,7 +295,10 @@ class DuplicateDetectorJob(RepairJob): 'artist_thumb_url': group[0].get('artist_thumb_url'), } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating duplicate finding: %s", e) result.errors += 1 diff --git a/core/repair_jobs/fake_lossless_detector.py b/core/repair_jobs/fake_lossless_detector.py index cc677ed3..dd8ffa5f 100644 --- a/core/repair_jobs/fake_lossless_detector.py +++ b/core/repair_jobs/fake_lossless_detector.py @@ -110,7 +110,7 @@ class FakeLosslessDetectorJob(RepairJob): log_type='error' ) if context.create_finding: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='fake_lossless', severity='warning', @@ -134,7 +134,10 @@ class FakeLosslessDetectorJob(RepairJob): 'file_size': os.path.getsize(fpath), } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error analyzing %s: %s", os.path.basename(fpath), e) diff --git a/core/repair_jobs/library_reorganize.py b/core/repair_jobs/library_reorganize.py index 3dc11245..6d7e68d2 100644 --- a/core/repair_jobs/library_reorganize.py +++ b/core/repair_jobs/library_reorganize.py @@ -1,247 +1,51 @@ """Library Reorganize Job — moves files to match the current file organization template. +Pre-rewrite this job had its own tag-reading + transfer-folder-walk + +template-application implementation that worked off file tags. The +classification heuristic ``is_album = (group_size > 1)`` (where +``group_size`` was the count of tracks for the same album currently +sitting in the transfer folder being scanned) misclassified album +tracks as singles whenever: + +- only one track of an album lived in the transfer folder (the rest + already moved to the library, or not yet downloaded), or +- album tags varied slightly across tracks (e.g. "Buds" vs + "Buds (Bonus)"), splitting them into separate single-element groups. + +Result: those tracks got routed through the SINGLE template and ended +up at e.g. ``Surf Curse/Surf Curse - Christine F/Surf Curse - Christine F.flac`` +instead of the album destination. + +GitHub issue #500 (@bafoed). Fix: delegate to the per-album planner +(``core.library_reorganize.preview_album_reorganize`` / +``reorganize_album``) the per-album reorganize modal already uses. The +planner is DB-driven — it knows the album has multiple tracks +regardless of how many currently sit in the transfer folder, so the +album-vs-single classification is structurally correct. + +Apply mode delegates to ``core.reorganize_queue`` so the actual file +move + post-processing + DB update + sidecar handling all flow +through the same code path the per-album modal uses. No second move +implementation to keep in sync. + Safety design: -- Dry run mode is ON by default. The job only creates findings (reports) showing - what WOULD move. The user must explicitly disable dry_run in job settings. -- The job is disabled by default (default_enabled=False) so it never runs - automatically unless the user explicitly enables it. -- Case-insensitive path comparison on Windows prevents false moves. -- Destination collision check prevents overwriting existing files. -- Files without usable tags are skipped, not guessed at. -- Moves are always within the transfer folder; cannot escape to parent dirs. +- Dry run mode is ON by default. Disabled by user explicitly. +- Job is disabled by default — never auto-runs unless user enables. +- Only DB-known tracks are considered. Files in transfer with no DB + entry are handled by the separate ``orphan_file_detector`` job. +- Albums with no matching metadata source ID are skipped with a + clear "needs enrichment first" finding rather than guessed at. """ -import json import os -import re -import shutil -import sys +from typing import Optional -from core.metadata_service import get_client_for_source, get_primary_source, get_source_priority from core.repair_jobs import register_job from core.repair_jobs.base import JobContext, JobResult, RepairJob from utils.logging_config import get_logger logger = get_logger("repair_job.library_reorganize") -AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'} -SIDECAR_EXTENSIONS = {'.lrc', '.jpg', '.jpeg', '.png', '.nfo', '.txt', '.cue'} -ALBUM_SIDECAR_NAMES = { - 'cover.jpg', 'cover.jpeg', 'cover.png', - 'folder.jpg', 'folder.jpeg', 'folder.png', - 'album.jpg', 'album.jpeg', 'album.png', - 'front.jpg', 'front.jpeg', 'front.png', - 'thumb.jpg', 'thumb.png', -} - -# Windows and macOS use case-insensitive filesystems by default -_CASE_INSENSITIVE = sys.platform in ('win32', 'darwin') - - -def _paths_equivalent(path_a: str, path_b: str) -> bool: - """Compare two paths, case-insensitive on Windows/macOS.""" - a = os.path.normpath(path_a) - b = os.path.normpath(path_b) - if _CASE_INSENSITIVE: - return a.lower() == b.lower() - return a == b - - -def _sanitize_filename(filename: str) -> str: - """Sanitize filename for file system compatibility.""" - sanitized = re.sub(r'[<>:"/\\|?*]', '_', filename) - sanitized = re.sub(r'\s+', ' ', sanitized).strip() - sanitized = sanitized.rstrip('. ') or '_' - if re.match(r'^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)', sanitized, re.IGNORECASE): - sanitized = '_' + sanitized - return sanitized[:200] - - -def _sanitize_context_values(context: dict) -> dict: - """Sanitize all string values for path safety. - - Empty strings are preserved so that template cleanup regexes can - remove surrounding decorators (e.g. ``($year)`` → ``()`` → removed). - """ - sanitized = {} - for key, value in context.items(): - if isinstance(value, str): - sanitized[key] = _sanitize_filename(value) if value else '' - else: - sanitized[key] = value - return sanitized - - -def _apply_path_template(template: str, context: dict) -> str: - """Apply template variables to build a path string.""" - clean = _sanitize_context_values(context) - # $cdnum — smart CD label. "CD01"/"CD02" only when multi-disc, empty for - # single-disc (collapses cleanly via the double-dash regex at the end). - _total_discs = int(clean.get('total_discs', 1) or 1) - _disc_number = int(clean.get('disc_number', 1) or 1) - cdnum_value = f"CD{_disc_number:02d}" if _total_discs > 1 else '' - - result = template - result = result.replace('${cdnum}', cdnum_value) - result = result.replace('$albumartist', clean.get('albumartist', clean.get('artist', 'Unknown Artist'))) - result = result.replace('$albumtype', clean.get('albumtype', 'Album')) - result = result.replace('$playlist', clean.get('playlist_name', '')) - result = result.replace('$artistletter', (clean.get('artist', 'U') or 'U')[0].upper()) - result = result.replace('$artist', clean.get('artist', 'Unknown Artist')) - result = result.replace('$album', clean.get('album', 'Unknown Album')) - result = result.replace('$title', clean.get('title', 'Unknown Track')) - result = result.replace('$cdnum', cdnum_value) - result = result.replace('$track', f"{clean.get('track_number', 1):02d}") - result = result.replace('$year', str(clean.get('year', ''))) - result = re.sub(r'\s+', ' ', result) - result = re.sub(r'\s*-\s*-\s*', ' - ', result) - return result.strip() - - -def _build_path_from_template(template: str, context: dict) -> tuple: - """Build (folder_path, filename_base) from a template string and context.""" - full_path = _apply_path_template(template, context) - quality_value = context.get('quality', '') - disc_value = f"{context.get('disc_number', 1):02d}" - - path_parts = full_path.split('/') - if len(path_parts) > 1: - folder_parts = path_parts[:-1] - filename_base = path_parts[-1] - - cleaned_folders = [] - for part in folder_parts: - part = part.replace('$quality', '') - part = part.replace('$disc', '') - part = re.sub(r'\s*\[\s*\]', '', part) - part = re.sub(r'\s*\(\s*\)', '', part) - part = re.sub(r'\s*\{\s*\}', '', part) - part = re.sub(r'\s*-\s*$', '', part) - part = re.sub(r'^\s*-\s*', '', part) - part = re.sub(r'\s+', ' ', part).strip() - if part: - cleaned_folders.append(part) - - filename_base = filename_base.replace('$quality', quality_value) - filename_base = filename_base.replace('$disc', disc_value) - filename_base = re.sub(r'\s*\[\s*\]', '', filename_base) - filename_base = re.sub(r'\s*\(\s*\)', '', filename_base) - filename_base = re.sub(r'\s*\{\s*\}', '', filename_base) - filename_base = re.sub(r'\s*-\s*$', '', filename_base) - # Leading dash cleanup — lets $cdnum at the start of a filename - # cleanly disappear on single-disc albums (empty-value case). - filename_base = re.sub(r'^\s*-\s*', '', filename_base) - filename_base = re.sub(r'\s+', ' ', filename_base).strip() - - sanitized_folders = [_sanitize_filename(p) for p in cleaned_folders] - folder_path = os.path.join(*sanitized_folders) if sanitized_folders else '' - return folder_path, _sanitize_filename(filename_base) - else: - full_path = full_path.replace('$quality', quality_value) - full_path = full_path.replace('$disc', disc_value) - full_path = re.sub(r'\s*\[\s*\]', '', full_path) - full_path = re.sub(r'\s*\(\s*\)', '', full_path) - full_path = re.sub(r'\s*\{\s*\}', '', full_path) - full_path = re.sub(r'\s*-\s*$', '', full_path) - full_path = re.sub(r'\s+', ' ', full_path).strip() - return '', _sanitize_filename(full_path) - - -def _get_audio_quality(file_path: str) -> str: - """Read audio file and return a quality descriptor string.""" - try: - ext = os.path.splitext(file_path)[1].lower() - if ext == '.flac': - from mutagen.flac import FLAC - audio = FLAC(file_path) - bits = audio.info.bits_per_sample - return f"FLAC {bits}bit" - elif ext == '.mp3': - from mutagen.mp3 import MP3, BitrateMode - audio = MP3(file_path) - kbps = audio.info.bitrate // 1000 - if audio.info.bitrate_mode == BitrateMode.VBR: - return "MP3-VBR" - return f"MP3-{kbps}" - elif ext in ('.m4a', '.aac', '.mp4'): - from mutagen.mp4 import MP4 - audio = MP4(file_path) - kbps = audio.info.bitrate // 1000 - return f"M4A-{kbps}" - elif ext == '.ogg': - from mutagen.oggvorbis import OggVorbis - audio = OggVorbis(file_path) - kbps = audio.info.bitrate // 1000 - return f"OGG-{kbps}" - elif ext == '.opus': - from mutagen.oggopus import OggOpus - audio = OggOpus(file_path) - kbps = audio.info.bitrate // 1000 - return f"OPUS-{kbps}" - return '' - except Exception: - return '' - - -def _read_tag_metadata(file_path: str) -> dict: - """Read artist, album, title, track_number, disc_number, year from file tags.""" - try: - from mutagen import File as MutagenFile - audio = MutagenFile(file_path, easy=True) - if audio is None: - return {} - - def first(tag_list): - if isinstance(tag_list, list) and tag_list: - return str(tag_list[0]) - if isinstance(tag_list, str): - return tag_list - return '' - - meta = {} - meta['artist'] = first(audio.get('artist', [''])) - meta['albumartist'] = first(audio.get('albumartist', [''])) or meta['artist'] - meta['album'] = first(audio.get('album', [''])) - meta['title'] = first(audio.get('title', [''])) - - # Track number: may be "3/12" format - raw_track = first(audio.get('tracknumber', ['1'])) - try: - meta['track_number'] = int(raw_track.split('/')[0]) - except (ValueError, IndexError): - meta['track_number'] = 1 - - # Disc number: may be "1/2" format - raw_disc = first(audio.get('discnumber', ['1'])) - try: - meta['disc_number'] = int(raw_disc.split('/')[0]) - except (ValueError, IndexError): - meta['disc_number'] = 1 - - # Year - raw_date = first(audio.get('date', [''])) - meta['year'] = raw_date[:4] if raw_date and len(raw_date) >= 4 else '' - - return meta - except Exception as e: - logger.debug("Failed to read tags from %s: %s", file_path, e) - return {} - - -def _remove_empty_dirs(directory: str, root: str): - """Remove empty directories up to root. Never removes root itself.""" - directory = os.path.normpath(directory) - root = os.path.normpath(root) - while directory != root and len(directory) > len(root) and directory.startswith(root): - try: - if os.path.isdir(directory) and not os.listdir(directory): - os.rmdir(directory) - directory = os.path.dirname(directory) - else: - break - except OSError: - break - @register_job class LibraryReorganizeJob(RepairJob): @@ -249,36 +53,36 @@ class LibraryReorganizeJob(RepairJob): display_name = 'Library Reorganize' description = 'Moves files to match the current file organization template (dry run by default)' help_text = ( - 'Scans your transfer folder and reads each audio file\'s tags (artist, album, title, ' - 'track number, disc number) to compute the expected file path based on your current ' - 'file organization template from Settings.\n\n' - 'Any file whose actual path doesn\'t match the expected template gets flagged. In dry ' - 'run mode (default), a finding is created showing the current and expected paths. ' - 'Disable dry run to have the job move files automatically.\n\n' - 'Safety features: case-insensitive path comparison on Windows/macOS, collision ' - 'detection, path escape prevention, and sidecar file handling (.lrc, .nfo, etc.).\n\n' + 'Iterates every album in your library and computes the expected file path for each ' + 'track using the same per-album planner the artist-detail "Reorganize" modal uses. ' + 'Any track whose current path doesn\'t match the expected path gets flagged in dry-run ' + 'mode or queued for a move in live mode.\n\n' + 'In live mode, moves are dispatched to the same reorganize queue the per-album modal ' + 'uses — file move + post-processing + DB update + sidecar handling all flow through ' + 'one code path.\n\n' + 'Albums with no matching metadata source ID are skipped — run enrichment first to ' + 'populate at least one of spotify_album_id / itunes_album_id / deezer_id.\n\n' + 'Files in the transfer folder that aren\'t tracked in the database are handled by ' + 'the separate Orphan File Detector job.\n\n' + 'Sidecars (.lrc, .jpg, .nfo, cover.jpg, etc) are handled by the underlying ' + 'reorganize queue: per-track sidecars are deleted at the source and album-level ' + 'cover art is re-downloaded fresh at the destination via the same post-processing ' + 'pipeline downloads use.\n\n' 'Settings:\n' - '- Dry Run: When enabled, only reports what would change without moving files\n' - '- Move Sidecars: Also move associated files (.lrc, .jpg, .nfo) alongside audio files' + '- Dry Run: When enabled, only reports what would change without moving files' ) icon = 'repair-icon-reorganize' default_enabled = False default_interval_hours = 168 # Weekly — but disabled by default so won't auto-run default_settings = { 'dry_run': True, - 'move_sidecars': True, } auto_fix = True def scan(self, context: JobContext) -> JobResult: result = JobResult() - transfer = context.transfer_folder - if not os.path.isdir(transfer): - logger.warning("Transfer folder does not exist: %s", transfer) - return result - - # Get template config cm = context.config_manager + if not cm: logger.error("No config manager available") return result @@ -286,600 +90,322 @@ class LibraryReorganizeJob(RepairJob): if not cm.get('file_organization.enabled', True): logger.info("File organization is disabled — skipping reorganize") if context.report_progress: - context.report_progress(phase='Skipped — file organization disabled', - log_line='File organization is disabled in settings', - log_type='skip') + context.report_progress( + phase='Skipped — file organization disabled', + log_line='File organization is disabled in settings', + log_type='skip', + ) return result - templates = cm.get('file_organization.templates', {}) - album_template = templates.get('album_path', '$albumartist/$albumartist - $album/$track - $title') - single_template = templates.get('single_path', '$artist/$artist - $title/$title') - disc_label = cm.get('file_organization.disc_label', 'Disc') - logger.info(f"Library Reorganize templates — album: '{album_template}', single: '{single_template}' (raw config: {templates})") - dry_run = self._get_setting(context, 'dry_run', True) - move_sidecars = self._get_setting(context, 'move_sidecars', True) + + # Imports kept inside scan() so the module can be imported in + # contexts that don't have web_server's Flask app booted. + from core.imports.paths import build_final_path_for_track + from core.library.path_resolver import resolve_library_file_path + from core.library_reorganize import preview_album_reorganize + + transfer_dir = context.transfer_folder + download_folder = cm.get('soulseek.download_path', '') if cm else '' + + def _resolve(file_path): + return resolve_library_file_path( + file_path, + transfer_folder=transfer_dir, + download_folder=download_folder, + config_manager=cm, + ) + + # Scope to the active media server only — the artist-detail + # reorganize modal does the same. Multi-server users (Plex + + # Jellyfin etc) shouldn't have this job touch the inactive + # server's files (different paths, likely shouldn't move). + active_server = None + try: + active_server = cm.get_active_media_server() + except Exception as exc: + logger.warning("Couldn't read active media server: %s", exc) + + album_rows = self._load_albums(context.db, active_server=active_server) + total = len(album_rows) + + if total == 0: + logger.info( + "No albums in DB to reorganize (active server: %s)", active_server, + ) + if context.report_progress: + context.report_progress( + phase='No albums to scan', + log_line=f'No albums for server "{active_server}" in database', + log_type='info', + ) + return result if context.report_progress: mode_label = 'DRY RUN' if dry_run else 'LIVE' - context.report_progress(phase=f'Scanning files ({mode_label})...', - log_line=f'Mode: {mode_label} — Scanning {transfer}', - log_type='info') + context.report_progress( + phase=f'Scanning {total} albums ({mode_label})...', + log_line=f'Mode: {mode_label} — Iterating {total} albums', + log_type='info', + scanned=0, total=total, + ) - # Collect all audio files - audio_files = [] - for root_dir, _dirs, files in os.walk(transfer): + items_to_enqueue = [] + + for i, album_row in enumerate(album_rows): if context.check_stop(): return result - for fname in files: - ext = os.path.splitext(fname)[1].lower() - if ext in AUDIO_EXTENSIONS: - audio_files.append(os.path.join(root_dir, fname)) - - total = len(audio_files) - if total == 0: - logger.info("No audio files found in transfer folder") - if context.report_progress: - context.report_progress(phase='No files found', log_line='No audio files in transfer folder', - log_type='info') - return result - - if context.report_progress: - context.report_progress(phase=f'Reading tags from {total} files...', - log_line=f'Found {total} audio files', - log_type='info', scanned=0, total=total) - - # Pre-read all tags and group by album for multi-disc detection - file_tags = {} # fpath -> tags dict - album_groups = {} # (albumartist, album) -> [fpath, ...] - for fpath in audio_files: - tags = _read_tag_metadata(fpath) - file_tags[fpath] = tags - key = (tags.get('albumartist', '') or tags.get('artist', ''), - tags.get('album', '')) - if key not in album_groups: - album_groups[key] = [] - album_groups[key].append(fpath) - - # Compute total_discs per album group - album_total_discs = {} - for key, fpaths in album_groups.items(): - max_disc = max((file_tags[fp].get('disc_number', 1) for fp in fpaths), default=1) - album_total_discs[key] = max_disc - - # Pre-load album years from DB for files missing year tags - db_album_years = {} # (artist, album) -> year string - needs_year = '$year' in (album_template + single_template) - if needs_year: - db_album_years = self._load_album_years(context.db) - - # API fallback: find (artist, album) pairs still missing year, batch-lookup - if needs_year and db_album_years is not None: - missing_pairs = set() - for _fpath, tags in file_tags.items(): - year = tags.get('year', '') - if year: - continue - artist = tags.get('artist', '') or tags.get('albumartist', '') - album = tags.get('album', '') or tags.get('title', '') - if not artist or not album: - continue - key = (artist.lower(), album.lower()) - if key not in db_album_years: - missing_pairs.add((artist, album)) - - if missing_pairs: - api_years = self._lookup_years_from_api(context, missing_pairs) - db_album_years.update(api_years) - - # Track claimed destinations to detect in-batch collisions - claimed_destinations = set() - # Track src_dir -> dest_dir for post-pass sidecar cleanup - moved_dirs = {} # src_dir -> dest_dir (last used destination) - - for i, fpath in enumerate(audio_files): - if context.check_stop(): - return result - if i % 50 == 0 and context.wait_if_paused(): + if i % 20 == 0 and context.wait_if_paused(): return result - result.scanned += 1 - fname = os.path.basename(fpath) - file_ext = os.path.splitext(fname)[1] + album_id = album_row['id'] + album_title = album_row['title'] or 'Unknown Album' - tags = file_tags.get(fpath, {}) - - # Skip files without minimum usable tags - title = tags.get('title', '') or '' - artist = tags.get('artist', '') or '' - if not title and not artist: - result.skipped += 1 - if context.report_progress and i % 20 == 0: - context.report_progress(scanned=i + 1, total=total, - phase=f'Processing ({i+1}/{total})...') + try: + preview = preview_album_reorganize( + album_id=str(album_id), + db=context.db, + transfer_dir=transfer_dir, + resolve_file_path_fn=_resolve, + build_final_path_fn=build_final_path_for_track, + ) + except Exception as exc: + logger.warning( + "Reorganize preview failed for album %s ('%s'): %s", + album_id, album_title, exc, + ) + result.errors += 1 continue - # Use defaults only when tags exist but are empty - artist = artist or 'Unknown Artist' - albumartist = tags.get('albumartist', '') or artist - album = tags.get('album', '') or '' - title = title or 'Unknown Track' - track_number = tags.get('track_number', 1) or 1 - disc_number = tags.get('disc_number', 1) or 1 - year = tags.get('year', '') + tracks = preview.get('tracks', []) + result.scanned += len(tracks) - # Fallback: if file tags have no year, try the DB album year - if not year and db_album_years: - year = db_album_years.get((artist.lower(), (album or title).lower()), '') - if not year: - year = db_album_years.get((albumartist.lower(), (album or title).lower()), '') + status = preview.get('status', '') - # Read quality for $quality template variable - quality = _get_audio_quality(fpath) - - # Determine template type: album or single - album_key = (albumartist, album) - group_size = len(album_groups.get(album_key, [])) - is_album = bool(album) and group_size > 1 - total_discs = album_total_discs.get(album_key, 1) - - template_context = { - 'artist': artist, - 'albumartist': albumartist, - 'album': album or title, - 'title': title, - 'track_number': track_number, - 'disc_number': disc_number, - # total_discs lets $cdnum decide whether to emit "CDxx" or - # stay empty (single-disc albums). - 'total_discs': total_discs, - 'year': year, - 'quality': quality, - 'albumtype': 'Album', - } - - if is_album: - template = album_template - user_controls_disc = '$disc' in template - folder_path, filename_base = _build_path_from_template(template, template_context) - if folder_path and filename_base: - if total_discs > 1 and not user_controls_disc: - disc_folder = f"{disc_label} {disc_number}" - expected = os.path.join(transfer, folder_path, disc_folder, filename_base + file_ext) - else: - expected = os.path.join(transfer, folder_path, filename_base + file_ext) - else: - result.skipped += 1 - continue - else: - template = single_template - folder_path, filename_base = _build_path_from_template(template, template_context) - if folder_path and filename_base: - expected = os.path.join(transfer, folder_path, filename_base + file_ext) - else: - result.skipped += 1 - continue - - # Safety: verify destination is still inside transfer folder - expected_norm = os.path.normpath(expected) - transfer_norm = os.path.normpath(transfer) - if not expected_norm.startswith(transfer_norm + os.sep) and expected_norm != transfer_norm: - logger.warning("Computed path escapes transfer folder, skipping: %s", expected_norm) + if status in ('no_album', 'no_tracks'): + # Album was deleted between the SELECT and the preview, + # or has no tracks. Nothing to do. result.skipped += 1 continue - actual_norm = os.path.normpath(fpath) - - # Case-insensitive comparison on Windows/macOS - if _paths_equivalent(actual_norm, expected_norm): - if context.report_progress and i % 20 == 0: - context.report_progress(scanned=i + 1, total=total, - phase=f'Processing ({i+1}/{total})...') - continue - - # Check for in-batch destination collision - dest_key = expected_norm.lower() if _CASE_INSENSITIVE else expected_norm - if dest_key in claimed_destinations: - result.skipped += 1 - if context.report_progress: - context.report_progress( - scanned=i + 1, total=total, - log_line=f'SKIP (duplicate dest): {os.path.basename(fpath)}', - log_type='skip' - ) - continue - claimed_destinations.add(dest_key) - - # File needs to move - if dry_run: - rel_actual = os.path.relpath(actual_norm, transfer) - rel_expected = os.path.relpath(expected_norm, transfer) - if context.create_finding: - context.create_finding( + if status == 'no_source_id': + # Can't compute destinations without a metadata source — + # skip cleanly with a single finding rather than 12 per-track + # "no source" findings that would clutter the UI. + result.skipped += len(tracks) or 1 + if dry_run and context.create_finding and tracks: + inserted = context.create_finding( job_id=self.job_id, - finding_type='path_mismatch', + finding_type='album_needs_enrichment', severity='info', - entity_type='file', - entity_id=None, - file_path=fpath, - title=f'Would move: {os.path.basename(fpath)}', - description=f'From: {rel_actual}\nTo: {rel_expected}', - details={'from': rel_actual, 'to': rel_expected} + entity_type='album', + entity_id=str(album_id), + file_path=None, + title=f'Needs enrichment: {album_title}', + description=( + f"Album '{album_title}' by {preview.get('artist', '?')} " + "has no metadata source ID — run enrichment first to " + "populate at least one of spotify_album_id / " + "itunes_album_id / deezer_id / discogs_id / soul_id." + ), + details={'album_id': str(album_id), 'reason': 'no_source_id'}, ) - result.findings_created += 1 - if context.report_progress: + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 + continue + + # Successful plan — count mismatched tracks + mismatched = [ + t for t in tracks + if t.get('matched') + and t.get('new_path') + and not t.get('unchanged') + and t.get('file_exists') + ] + + if not mismatched: + if context.report_progress and (i + 1) % 25 == 0: context.report_progress( scanned=i + 1, total=total, - phase=f'Dry run ({i+1}/{total})...', - log_line=f'[DRY] {os.path.basename(fpath)} -> {os.path.relpath(expected_norm, transfer)}', - log_type='info' + phase=f'Scanning ({i+1}/{total})...', ) + continue + + if dry_run: + # One finding per track that would move. + for t in mismatched: + try: + if context.create_finding: + inserted = context.create_finding( + job_id=self.job_id, + finding_type='path_mismatch', + severity='info', + entity_type='track', + entity_id=str(t.get('track_id') or ''), + file_path=t.get('current_path') or '', + title=f"Would move: {os.path.basename(t.get('current_path') or '') or t.get('title', '')}", + description=( + f"From: {t.get('current_path') or '?'}\n" + f"To: {t.get('new_path') or '?'}" + ), + details={ + 'from': t.get('current_path') or '', + 'to': t.get('new_path') or '', + 'album_id': str(album_id), + 'album_title': album_title, + 'source': preview.get('source'), + 'track_id': t.get('track_id'), + }, + ) + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 + except Exception as e: + logger.debug( + "Error creating path_mismatch finding for track %s: %s", + t.get('track_id'), e, + ) + result.errors += 1 else: - # Actually move the file - try: - dest_dir = os.path.dirname(expected_norm) - os.makedirs(dest_dir, exist_ok=True) + # Apply mode: enqueue the album for the live reorganize + # queue worker. The queue handles file move + post-process + # + DB update + sidecar via the same code path the per- + # album modal uses — no second move implementation. + items_to_enqueue.append({ + 'album_id': str(album_id), + 'album_title': album_title, + 'artist_id': str(album_row.get('artist_id') or ''), + 'artist_name': preview.get('artist') or album_row.get('artist_name') or 'Unknown Artist', + 'source': preview.get('source'), + }) - # Collision: skip if destination already exists and is a different file - if os.path.exists(expected_norm): - # On case-insensitive FS, check if it's the same file (case rename) - try: - same_file = os.path.samefile(actual_norm, expected_norm) - except (OSError, ValueError): - same_file = False - - if not same_file: - result.skipped += 1 - if context.report_progress: - context.report_progress( - scanned=i + 1, total=total, - log_line=f'SKIP (exists): {os.path.basename(fpath)}', - log_type='skip' - ) - continue - - # Same file, different case — use two-step rename to avoid - # OS refusing rename to "same" path on case-insensitive FS - if _CASE_INSENSITIVE: - tmp_path = expected_norm + '.tmp_rename' - shutil.move(actual_norm, tmp_path) - shutil.move(tmp_path, expected_norm) - else: - shutil.move(actual_norm, expected_norm) - else: - shutil.move(actual_norm, expected_norm) - - result.auto_fixed += 1 - # Record src->dest for post-pass sidecar cleanup - # For multi-disc, use album root (parent of Disc N/) as destination - sidecar_dest = dest_dir - if is_album and total_discs > 1 and '$disc' not in album_template: - sidecar_dest = os.path.dirname(dest_dir) - moved_dirs[os.path.dirname(actual_norm)] = sidecar_dest - - # Move sidecar files (LRC, cover art, etc.) - if move_sidecars: - stem = os.path.splitext(os.path.basename(actual_norm))[0] - src_dir = os.path.dirname(actual_norm) - # Track-level sidecars (same stem as audio file) - for sidecar_ext in SIDECAR_EXTENSIONS: - sidecar_src = os.path.join(src_dir, stem + sidecar_ext) - if os.path.isfile(sidecar_src): - new_stem = os.path.splitext(os.path.basename(expected_norm))[0] - sidecar_dst = os.path.join(dest_dir, new_stem + sidecar_ext) - try: - shutil.move(sidecar_src, sidecar_dst) - except Exception as se: - logger.debug("Failed to move sidecar %s: %s", sidecar_src, se) - # Album-level sidecars (cover.jpg, folder.jpg, etc.) - # For multi-disc, place in the album root (parent of Disc N/) - album_dest = dest_dir - if is_album and total_discs > 1 and '$disc' not in album_template: - album_dest = os.path.dirname(dest_dir) - os.makedirs(album_dest, exist_ok=True) - for album_sidecar in ALBUM_SIDECAR_NAMES: - sidecar_src = os.path.join(src_dir, album_sidecar) - if os.path.isfile(sidecar_src): - sidecar_dst = os.path.join(album_dest, album_sidecar) - if not os.path.exists(sidecar_dst): - try: - shutil.move(sidecar_src, sidecar_dst) - except Exception as se: - logger.debug("Failed to move album sidecar %s: %s", sidecar_src, se) - - # Update DB file_path if there's a matching track - self._update_db_path(context.db, actual_norm, expected_norm, transfer) - - # Clean up empty source directories - _remove_empty_dirs(os.path.dirname(actual_norm), transfer) - - if context.report_progress: - context.report_progress( - scanned=i + 1, total=total, - phase=f'Moving ({i+1}/{total})...', - log_line=f'Moved: {os.path.basename(fpath)}', - log_type='success' - ) - except Exception as e: - logger.error("Failed to move %s -> %s: %s", fpath, expected_norm, e) - result.errors += 1 - if context.report_progress: - context.report_progress( - scanned=i + 1, total=total, - log_line=f'ERROR: {os.path.basename(fpath)} -- {e}', - log_type='error' - ) - - if context.update_progress and (i + 1) % 10 == 0: + if context.update_progress and (i + 1) % 25 == 0: context.update_progress(i + 1, total) + if context.report_progress and (i + 1) % 25 == 0: + context.report_progress( + scanned=i + 1, total=total, + phase=f'{"Dry run" if dry_run else "Queueing"} ({i+1}/{total})...', + ) + + # Bulk enqueue collected items in one batch (apply mode only). + # Queue's tally shape: {'enqueued': N, 'already_queued': M, 'total': K}. + if items_to_enqueue: + try: + from core.reorganize_queue import get_queue + queue_summary = get_queue().enqueue_many(items_to_enqueue) + enqueued_count = queue_summary.get('enqueued', 0) + already_queued = queue_summary.get('already_queued', 0) + result.auto_fixed += enqueued_count + # Dedupe-skipped albums are tracked separately so the + # repair-job summary doesn't double-count. + result.skipped += already_queued + logger.info( + "Reorganize: enqueued %d albums (%d already in queue)", + enqueued_count, already_queued, + ) + except Exception as exc: + logger.error("Failed to enqueue reorganize items: %s", exc) + result.errors += 1 if context.update_progress: context.update_progress(total, total) - # Post-pass: move leftover sidecar files from directories that lost all audio - if not dry_run and move_sidecars and moved_dirs: - for src_dir, dest_dir in moved_dirs.items(): - if not os.path.isdir(src_dir): - continue - # Check if any audio files remain in this directory - has_audio = any( - os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS - for f in os.listdir(src_dir) - if os.path.isfile(os.path.join(src_dir, f)) - ) - if has_audio: - continue - # Move all remaining sidecar-type files to the destination - for f in os.listdir(src_dir): - fpath_full = os.path.join(src_dir, f) - if not os.path.isfile(fpath_full): - continue - ext = os.path.splitext(f)[1].lower() - if ext in SIDECAR_EXTENSIONS: - dst = os.path.join(dest_dir, f) - if not os.path.exists(dst): - try: - shutil.move(fpath_full, dst) - except Exception: - pass - # Try cleaning up the now-potentially-empty directory - _remove_empty_dirs(src_dir, transfer) - - mode_text = 'Dry run' if dry_run else 'Reorganize' - summary = f"{mode_text} complete: {result.scanned} scanned, {result.auto_fixed} moved, {result.findings_created} findings, {result.skipped} skipped, {result.errors} errors" + mode_text = 'Dry run' if dry_run else 'Enqueue' + summary = ( + f"{mode_text} complete: {result.scanned} tracks scanned across {total} albums, " + f"{result.auto_fixed} albums queued, {result.findings_created} findings, " + f"{result.skipped} skipped, {result.errors} errors" + ) logger.info(summary) if context.report_progress: context.report_progress( phase='Complete', log_line=summary, log_type='success', - scanned=total, total=total + scanned=total, total=total, ) return result def estimate_scope(self, context: JobContext) -> int: - transfer = context.transfer_folder - if not os.path.isdir(transfer): + """Estimate is the active-server album count — matches what + scan() iterates over.""" + active_server = None + if context.config_manager: + try: + active_server = context.config_manager.get_active_media_server() + except Exception as e: + logger.debug("active media server lookup: %s", e) + try: + conn = context.db._get_connection() + try: + cursor = conn.cursor() + if active_server: + cursor.execute( + "SELECT COUNT(*) FROM albums WHERE server_source = ?", + (active_server,), + ) + else: + cursor.execute("SELECT COUNT(*) FROM albums") + row = cursor.fetchone() + return int(row[0]) if row else 0 + finally: + conn.close() + except Exception: return 0 - count = 0 - for _root, _dirs, files in os.walk(transfer): - for fname in files: - if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS: - count += 1 - return count + + def _load_albums(self, db, active_server: Optional[str] = None) -> list: + """Load minimal album metadata (id, title, artist_id, artist_name) + for albums on the active media server. + + SoulSync's DB stores rows for every configured server (Plex + + Jellyfin + Navidrome + SoulSync standalone) distinguished by + the ``server_source`` column. The artist-detail reorganize + modal only sees the active server's library; this job matches + that scope so users don't accidentally try to reorganize the + inactive server's files (which live at different paths and + likely shouldn't move). + + ``active_server=None`` falls back to "no filter" — used by + legacy callers / tests that don't have a config_manager. + """ + conn = None + try: + conn = db._get_connection() + cursor = conn.cursor() + if active_server: + cursor.execute(""" + SELECT al.id, al.title, al.artist_id, ar.name AS artist_name + FROM albums al + LEFT JOIN artists ar ON ar.id = al.artist_id + WHERE al.server_source = ? + ORDER BY al.id + """, (active_server,)) + else: + cursor.execute(""" + SELECT al.id, al.title, al.artist_id, ar.name AS artist_name + FROM albums al + LEFT JOIN artists ar ON ar.id = al.artist_id + ORDER BY al.id + """) + return [dict(row) for row in cursor.fetchall()] + except Exception as e: + logger.error("Failed to load album list for reorganize: %s", e) + return [] + finally: + if conn: + try: + conn.close() + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down + pass def _get_setting(self, context: JobContext, key: str, default): """Read a job-specific setting from config.""" if context.config_manager: - return context.config_manager.get(f'repair.jobs.{self.job_id}.settings.{key}', default) - return default - - def _load_album_years(self, db) -> dict: - """Load all album years from DB. Returns {(artist_lower, album_lower): year_str}. - - Checks the main albums table first, then falls back to discovery_pool - release_date for tracks that were playlist-synced without year metadata. - """ - years = {} - conn = None - try: - conn = db._get_connection() - cursor = conn.cursor() - - # Source 1: albums table (most authoritative) - cursor.execute(""" - SELECT ar.name, al.title, al.year - FROM albums al - JOIN artists ar ON ar.id = al.artist_id - WHERE al.year IS NOT NULL AND al.year != 0 - """) - for row in cursor.fetchall(): - artist_name, album_title, year = row - if artist_name and album_title and year: - key = (artist_name.lower(), album_title.lower()) - years[key] = str(year)[:4] - - # Source 2: discovery_pool release_date (covers playlist-synced tracks) - try: - cursor.execute(""" - SELECT artist_name, album_name, release_date - FROM discovery_pool - WHERE release_date IS NOT NULL AND release_date != '' - """) - for row in cursor.fetchall(): - artist_name, album_name, release_date = row - if artist_name and album_name and release_date: - key = (artist_name.lower(), album_name.lower()) - if key not in years: # Don't override albums table - year_str = str(release_date)[:4] - if len(year_str) == 4 and year_str.isdigit(): - years[key] = year_str - except Exception: - pass # discovery_pool may not exist on all installs - - # Source 3: recent_releases (watchlist artist releases) - try: - cursor.execute(""" - SELECT wa.artist_name, rr.album_name, rr.release_date - FROM recent_releases rr - JOIN watchlist_artists wa ON wa.id = rr.watchlist_artist_id - WHERE rr.release_date IS NOT NULL AND rr.release_date != '' - """) - for row in cursor.fetchall(): - artist_name, album_name, release_date = row - if artist_name and album_name and release_date: - key = (artist_name.lower(), album_name.lower()) - if key not in years: - year_str = str(release_date)[:4] - if len(year_str) == 4 and year_str.isdigit(): - years[key] = year_str - except Exception: - pass # recent_releases may not exist on all installs - - # Source 4: wishlist tracks (spotify_data JSON contains release date) - try: - cursor.execute(""" - SELECT spotify_data FROM wishlist_tracks - WHERE spotify_data IS NOT NULL AND spotify_data != '' - """) - for row in cursor.fetchall(): - try: - data = json.loads(row[0]) - artist_name = '' - if data.get('artists'): - a = data['artists'][0] - artist_name = a.get('name', '') if isinstance(a, dict) else str(a) - album_data = data.get('album', {}) - album_name = album_data.get('name', '') if isinstance(album_data, dict) else '' - release_date = album_data.get('release_date', '') if isinstance(album_data, dict) else '' - if artist_name and album_name and release_date: - key = (artist_name.lower(), album_name.lower()) - if key not in years: - year_str = str(release_date)[:4] - if len(year_str) == 4 and year_str.isdigit(): - years[key] = year_str - except (ValueError, KeyError, TypeError): - continue - except Exception: - pass # wishlist_tracks may not exist or have different schema - - except Exception as e: - logger.debug("Failed to load album years from DB: %s", e) - finally: - if conn: - conn.close() - return years - - def _lookup_years_from_api(self, context, missing_pairs) -> dict: - """Batch-lookup release years from the configured metadata providers for albums not found in DB. - - Args: - context: JobContext with config_manager - missing_pairs: set of (artist, album) tuples needing year lookup - - Returns: - dict of {(artist_lower, album_lower): year_str} - """ - years = {} - if not missing_pairs: - return years - - primary_source = get_primary_source() - source_priority = get_source_priority(primary_source) - - # Cap lookups to avoid excessive API calls - max_lookups = 200 - pairs_list = list(missing_pairs)[:max_lookups] - logger.info("Looking up %d album years from configured metadata providers", len(pairs_list)) - - if context.report_progress: - context.report_progress( - phase=f'Looking up {len(pairs_list)} album years from metadata providers...', - log_line=f'Fetching release years for {len(pairs_list)} albums', - log_type='info' + return context.config_manager.get( + f'repair.jobs.{self.job_id}.settings.{key}', default, ) - - for artist, album in pairs_list: - if context.check_stop(): - break - key = (artist.lower(), album.lower()) - for source_name in source_priority: - try: - search_client = get_client_for_source(source_name) - if not search_client or not hasattr(search_client, 'search_albums'): - continue - results = search_client.search_albums(f"{artist} {album}", limit=3) - if results: - for r in results: - release_date = getattr(r, 'release_date', '') or '' - if release_date and len(release_date) >= 4: - year_str = release_date[:4] - if year_str.isdigit(): - years[key] = year_str - break - if key in years: - break - if context.sleep_or_stop(0.1): # Rate limit courtesy - return years - except Exception as e: - logger.debug("API year lookup failed for %s - %s via %s: %s", artist, album, source_name, e) - - logger.info("API year lookup: found %d/%d years", len(years), len(pairs_list)) - return years - - def _update_db_path(self, db, old_path: str, new_path: str, transfer_folder: str = ''): - """Update file_path in the tracks table when a file is moved. - - DB may store server-side paths (e.g. /mnt/musicBackup/Artist/Album/track.flac) - while local paths use the transfer folder (e.g. H:\\Music\\Artist\\Album\\track.flac). - Falls back to suffix matching when exact match fails. - """ - conn = None - try: - conn = db._get_connection() - cursor = conn.cursor() - - # Try exact match first - cursor.execute("UPDATE tracks SET file_path = ? WHERE file_path = ?", - (new_path, old_path)) - if cursor.rowcount > 0: - conn.commit() - return - - # Try normalized path match - cursor.execute("UPDATE tracks SET file_path = ? WHERE file_path = ?", - (new_path, os.path.normpath(old_path))) - if cursor.rowcount > 0: - conn.commit() - return - - # Suffix match: compute path relative to transfer folder and match - # against DB paths that may use a different base prefix - if transfer_folder: - try: - rel_suffix = os.path.relpath(old_path, transfer_folder).replace('\\', '/') - # Escape LIKE wildcards (% _ ^) so artist/album names are literal - escaped = rel_suffix.replace('^', '^^').replace('%', '^%').replace('_', '^_') - cursor.execute( - "UPDATE tracks SET file_path = ? WHERE file_path LIKE ? ESCAPE '^'", - (new_path, '%/' + escaped) - ) - if cursor.rowcount > 0: - conn.commit() - return - # Also try with backslash separators (Windows DB paths) - escaped_bs = escaped.replace('/', '\\') - cursor.execute( - "UPDATE tracks SET file_path = ? WHERE file_path LIKE ? ESCAPE '^'", - (new_path, '%\\' + escaped_bs) - ) - except Exception: - pass - - conn.commit() - except Exception as e: - logger.debug("DB path update failed for %s: %s", old_path, e) - finally: - if conn: - conn.close() + return default diff --git a/core/repair_jobs/live_commentary_cleaner.py b/core/repair_jobs/live_commentary_cleaner.py index b3532567..60de59b4 100644 --- a/core/repair_jobs/live_commentary_cleaner.py +++ b/core/repair_jobs/live_commentary_cleaner.py @@ -207,7 +207,7 @@ class LiveCommentaryCleanerJob(RepairJob): if context.create_finding: try: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='unwanted_content', severity='info', @@ -239,7 +239,10 @@ class LiveCommentaryCleanerJob(RepairJob): 'artist_thumb_url': artist_thumb or None, } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating finding: %s", e) result.errors += 1 diff --git a/core/repair_jobs/lossy_converter.py b/core/repair_jobs/lossy_converter.py index 3d9e12ef..da1f1646 100644 --- a/core/repair_jobs/lossy_converter.py +++ b/core/repair_jobs/lossy_converter.py @@ -7,6 +7,7 @@ ffmpeg with the user's configured codec/bitrate settings. import os +from core.library.path_resolver import resolve_library_file_path from core.repair_jobs import register_job from core.repair_jobs.base import JobContext, JobResult, RepairJob from utils.logging_config import get_logger @@ -20,21 +21,14 @@ CODEC_MAP = { } -def _resolve_file_path(file_path, transfer_folder, download_folder=None): - """Resolve a stored DB path to an actual file on disk.""" - if not file_path: - return None - if os.path.exists(file_path): - return file_path - path_parts = file_path.replace('\\', '/').split('/') - for base_dir in [transfer_folder, download_folder]: - if not base_dir or not os.path.isdir(base_dir): - continue - for i in range(1, len(path_parts)): - candidate = os.path.join(base_dir, *path_parts[i:]) - if os.path.exists(candidate): - return candidate - return None +def _resolve_file_path(file_path, transfer_folder, download_folder=None, config_manager=None): + """Backwards-compat wrapper. Use ``resolve_library_file_path`` directly.""" + return resolve_library_file_path( + file_path, + transfer_folder=transfer_folder, + download_folder=download_folder, + config_manager=config_manager, + ) @register_job @@ -136,7 +130,8 @@ class LossyConverterJob(RepairJob): ) # Resolve path - resolved = _resolve_file_path(file_path, context.transfer_folder, download_folder) + resolved = _resolve_file_path(file_path, context.transfer_folder, download_folder, + config_manager=context.config_manager) if not resolved or not os.path.exists(resolved): continue @@ -155,7 +150,7 @@ class LossyConverterJob(RepairJob): if context.create_finding: try: file_size = os.path.getsize(resolved) - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='missing_lossy_copy', severity='info', @@ -182,7 +177,10 @@ class LossyConverterJob(RepairJob): 'artist_thumb_url': artist_thumb or None, } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating finding for track %s: %s", track_id, e) result.errors += 1 diff --git a/core/repair_jobs/mbid_mismatch_detector.py b/core/repair_jobs/mbid_mismatch_detector.py index 8d24835a..e35e9683 100644 --- a/core/repair_jobs/mbid_mismatch_detector.py +++ b/core/repair_jobs/mbid_mismatch_detector.py @@ -8,15 +8,19 @@ SoulSync shows them correctly. """ import os +from collections import Counter, defaultdict from difflib import SequenceMatcher +from typing import Optional +from core.library.path_resolver import resolve_library_file_path from core.repair_jobs import register_job from core.repair_jobs.base import JobContext, JobResult, RepairJob from utils.logging_config import get_logger logger = get_logger("repair_job.mbid_mismatch") -# Tag name → format mappings (must match web_server.py write logic) +# Tag name → format mappings for the TRACK MBID (existing detection). +# Must match web_server.py write logic. _MBID_TAG_KEYS = { # MP3 (ID3): UFID frame with owner 'http://musicbrainz.org' 'mp3_ufid_owner': 'http://musicbrainz.org', @@ -26,6 +30,14 @@ _MBID_TAG_KEYS = { 'mp4': '----:com.apple.iTunes:MusicBrainz Track Id', } +# Tag name → format mappings for the ALBUM MBID (new in this PR). +# Same Picard standards as `core/metadata/source.py:ID3_TAG_MAP` etc. +_ALBUM_MBID_TAG_KEYS = { + 'mp3_txxx_desc': 'MusicBrainz Album Id', # ID3 TXXX frame description + 'vorbis': 'MUSICBRAINZ_ALBUMID', # FLAC/OGG vorbis comment + 'mp4': '----:com.apple.iTunes:MusicBrainz Album Id', +} + TITLE_SIMILARITY_THRESHOLD = 0.55 @@ -165,22 +177,123 @@ def _remove_mbid_from_file(file_path): return False -def _resolve_file_path(file_path, transfer_folder, download_folder=None): - """Resolve a stored DB path to an actual file on disk.""" - if not file_path: - return None - if os.path.exists(file_path): - return file_path +def _read_album_mbid_from_file(file_path: str) -> Optional[str]: + """Read the embedded MusicBrainz Album Id from an audio file's tags. - path_parts = file_path.replace('\\', '/').split('/') - for base_dir in [transfer_folder, download_folder]: - if not base_dir or not os.path.isdir(base_dir): - continue - for i in range(1, len(path_parts)): - candidate = os.path.join(base_dir, *path_parts[i:]) - if os.path.exists(candidate): - return candidate - return None + Mirrors `_read_file_tags` but for the ALBUM MBID. Returns None when + the file has no album MBID tag, when the file is unreadable, or + when the tag format is unsupported. + """ + try: + from mutagen import File as MutagenFile + from mutagen.id3 import ID3 + from mutagen.flac import FLAC + from mutagen.oggvorbis import OggVorbis + from mutagen.mp4 import MP4 + + audio = MutagenFile(file_path) + if audio is None: + return None + + if isinstance(audio.tags, ID3): + txxx_key = f'TXXX:{_ALBUM_MBID_TAG_KEYS["mp3_txxx_desc"]}' + tag = audio.tags.get(txxx_key) + if tag and tag.text: + value = tag.text[0] + return str(value).strip() if value else None + # Some taggers use lowercase variant + txxx_key_lower = 'TXXX:MUSICBRAINZ_ALBUMID' + tag = audio.tags.get(txxx_key_lower) + if tag and tag.text: + value = tag.text[0] + return str(value).strip() if value else None + return None + + if isinstance(audio, (FLAC, OggVorbis)): + for key in (_ALBUM_MBID_TAG_KEYS['vorbis'], 'musicbrainz_albumid'): + vals = audio.get(key, []) + if vals: + value = vals[0] + return str(value).strip() if value else None + return None + + if isinstance(audio, MP4): + mp4_key = _ALBUM_MBID_TAG_KEYS['mp4'] + vals = audio.get(mp4_key, []) + if vals: + raw = vals[0] + value = raw.decode('utf-8', errors='ignore') if isinstance(raw, bytes) else str(raw) + return value.strip() if value else None + return None + + return None + except Exception as e: + logger.debug("Error reading album MBID from %s: %s", file_path, e) + return None + + +def _write_album_mbid_to_file(file_path: str, new_mbid: str) -> bool: + """Rewrite the embedded MusicBrainz Album Id tag. + + Used by the fix action to bring a track's album MBID in line with + the consensus across other tracks of the same album. Returns True + when the tag was written and the file saved. + """ + if not new_mbid: + return False + try: + from mutagen import File as MutagenFile + from mutagen.id3 import ID3, TXXX + from mutagen.flac import FLAC + from mutagen.oggvorbis import OggVorbis + from mutagen.mp4 import MP4 + + audio = MutagenFile(file_path) + if audio is None: + return False + + if isinstance(audio.tags, ID3): + # Wipe any conflicting variants first so we don't end up with + # two TXXX frames pointing at different MBIDs. + for key in ('TXXX:MUSICBRAINZ_ALBUMID', + f'TXXX:{_ALBUM_MBID_TAG_KEYS["mp3_txxx_desc"]}'): + if key in audio.tags: + del audio.tags[key] + audio.tags.add(TXXX( + encoding=3, + desc=_ALBUM_MBID_TAG_KEYS['mp3_txxx_desc'], + text=[new_mbid], + )) + audio.save() + return True + + if isinstance(audio, (FLAC, OggVorbis)): + for key in ('musicbrainz_albumid',): + if key in audio: + del audio[key] + audio[_ALBUM_MBID_TAG_KEYS['vorbis']] = [new_mbid] + audio.save() + return True + + if isinstance(audio, MP4): + audio[_ALBUM_MBID_TAG_KEYS['mp4']] = [new_mbid.encode('utf-8')] + audio.save() + return True + + return False + except Exception as e: + logger.error("Error writing album MBID to %s: %s", file_path, e) + return False + + +def _resolve_file_path(file_path, transfer_folder, download_folder=None, config_manager=None): + """Backwards-compat wrapper. Use ``resolve_library_file_path`` directly.""" + return resolve_library_file_path( + file_path, + transfer_folder=transfer_folder, + download_folder=download_folder, + config_manager=config_manager, + ) @register_job @@ -253,8 +366,8 @@ class MbidMismatchDetectorJob(RepairJob): try: from core.musicbrainz_client import MusicBrainzClient mb_client = MusicBrainzClient() - except Exception: - pass + except Exception as e: + logger.debug("MusicBrainz client init failed: %s", e) if not mb_client: logger.warning("MusicBrainz client not available, skipping MBID mismatch scan") @@ -280,7 +393,8 @@ class MbidMismatchDetectorJob(RepairJob): context.update_progress(i + 1, total) # Resolve the file path - resolved = _resolve_file_path(file_path, context.transfer_folder, download_folder) + resolved = _resolve_file_path(file_path, context.transfer_folder, download_folder, + config_manager=context.config_manager) if not resolved: result.scanned += 1 continue @@ -349,19 +463,200 @@ class MbidMismatchDetectorJob(RepairJob): if context.update_progress: context.update_progress(total, total) - logger.info("MBID mismatch scan: %d files scanned, %d with MBIDs verified, %d mismatches found", + logger.info("MBID mismatch scan (track-level): %d files scanned, %d with MBIDs verified, %d mismatches found", total, checked, result.findings_created) + # Phase 2: Album MBID consistency check. + # + # Tracks of the same album that carry different MUSICBRAINZ_ALBUMID + # tags cause Navidrome (and other media servers grouping by album + # MBID) to split the album into multiple entries. Reported by user + # Samuel [KC]. Detection strategy: group tracks by DB album_id, + # find the consensus (most-common) album MBID, flag the dissenters. + # No MusicBrainz API calls — this is a pure consistency check, so + # it doesn't compete with the rate-limited track scan above. + track_findings_so_far = result.findings_created + self._scan_album_mbid_consistency(context, result, download_folder) + album_findings = result.findings_created - track_findings_so_far + if context.report_progress: context.report_progress( scanned=total, total=total, phase='Complete', - log_line=f'Verified {checked} MBIDs — {result.findings_created} mismatches found', + log_line=( + f'Verified {checked} track MBIDs ({track_findings_so_far} mismatches) — ' + f'album consistency check found {album_findings} dissenters' + ), log_type='success' if result.findings_created == 0 else 'warning' ) return result + def _scan_album_mbid_consistency(self, context: JobContext, result: JobResult, + download_folder: str) -> None: + """Group tracks by DB album, flag tracks whose embedded album + MBID differs from the consensus across the album's other tracks.""" + # Pull tracks grouped by album. Singles (album NULL) skipped — they + # can't have a consistency issue. + rows = [] + conn = None + try: + conn = context.db._get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT t.id, t.title, t.album_id, t.file_path, + ar.name AS artist_name, al.title AS album_title, + al.thumb_url AS album_thumb, ar.thumb_url AS artist_thumb + FROM tracks t + LEFT JOIN artists ar ON ar.id = t.artist_id + LEFT JOIN albums al ON al.id = t.album_id + WHERE t.file_path IS NOT NULL AND t.file_path != '' + AND t.album_id IS NOT NULL + """) + rows = cursor.fetchall() + except Exception as e: + logger.error("Album MBID consistency scan: DB fetch failed: %s", e, exc_info=True) + return + finally: + if conn: + conn.close() + + if not rows: + return + + # Group by album_id. Read each track's embedded album MBID — only + # include rows where the read succeeded (skips files that don't + # have an album MBID at all; those don't break Navidrome since + # there's no MBID for it to disagree on). + by_album: dict = defaultdict(list) + for row in rows: + if context.check_stop(): + return + track_id = row['id'] + album_id = row['album_id'] + file_path = row['file_path'] + + resolved = _resolve_file_path( + file_path, context.transfer_folder, download_folder, + config_manager=context.config_manager, + ) + if not resolved: + continue + album_mbid = _read_album_mbid_from_file(resolved) + if not album_mbid: + continue + by_album[album_id].append({ + 'track_id': track_id, + 'title': row['title'], + 'album_title': row['album_title'], + 'artist_name': row['artist_name'], + 'album_thumb': row['album_thumb'], + 'artist_thumb': row['artist_thumb'], + 'file_path': file_path, + 'resolved': resolved, + 'album_mbid': album_mbid, + }) + + if context.report_progress: + context.report_progress( + phase=f'Checking album MBID consistency across {len(by_album)} albums...', + log_type='info', + ) + + for album_id, tracks_in_album in by_album.items(): + if context.check_stop(): + return + # Need at least 2 tracks to detect a mismatch. + if len(tracks_in_album) < 2: + continue + + mbid_counts = Counter(t['album_mbid'] for t in tracks_in_album) + if len(mbid_counts) == 1: + continue # All tracks agree → nothing to flag + + consensus_mbid, consensus_count = mbid_counts.most_common(1)[0] + + # Defensive: if no MBID has a clear plurality (e.g. 3 tracks, + # 3 different MBIDs), skip rather than picking a random one. + # Counter.most_common returns ties in arbitrary order; we don't + # want to fix a track to a "consensus" that's really a 1/N tie. + second_count = mbid_counts.most_common(2)[1][1] if len(mbid_counts) > 1 else 0 + if consensus_count == second_count: + logger.info( + "Album %s has tied album MBID counts %s — no clear consensus, skipping", + album_id, dict(mbid_counts), + ) + continue + + for track in tracks_in_album: + if track['album_mbid'] == consensus_mbid: + continue + self._create_album_mbid_mismatch_finding( + context, result, track, + consensus_mbid=consensus_mbid, + consensus_count=consensus_count, + total_tracks=len(tracks_in_album), + ) + + def _create_album_mbid_mismatch_finding(self, context: JobContext, result: JobResult, + track: dict, consensus_mbid: str, + consensus_count: int, total_tracks: int) -> None: + """Create a finding for a track whose album MBID disagrees with + the consensus across the album's other tracks.""" + title = track['title'] + artist_name = track['artist_name'] + album_title = track['album_title'] + if context.report_progress: + context.report_progress( + log_line=( + f'Album MBID mismatch: "{title}" — has {track["album_mbid"][:8]}…, ' + f'consensus is {consensus_mbid[:8]}… ({consensus_count}/{total_tracks} tracks)' + ), + log_type='warning' + ) + if context.create_finding: + try: + inserted = context.create_finding( + job_id=self.job_id, + finding_type='album_mbid_mismatch', + severity='warning', + entity_type='track', + entity_id=str(track['track_id']), + file_path=track['file_path'], + title=f'Album MBID mismatch: {title or "Unknown"}', + description=( + f'Track "{title}" by {artist_name or "Unknown"} on album ' + f'"{album_title or "Unknown"}" has a different ' + f'MusicBrainz Album Id than the album\'s other tracks. ' + f'This causes media servers like Navidrome to split the album.' + ), + details={ + 'track_id': track['track_id'], + 'title': title, + 'artist': artist_name, + 'album': album_title, + 'file_path': track['file_path'], + 'wrong_mbid': track['album_mbid'], + 'consensus_mbid': consensus_mbid, + 'consensus_count': consensus_count, + 'total_tracks_with_mbid': total_tracks, + 'reason': ( + f'{consensus_count}/{total_tracks} tracks of this album use ' + f'MBID {consensus_mbid}; this track uses {track["album_mbid"]}' + ), + 'album_thumb_url': track['album_thumb'] or None, + 'artist_thumb_url': track['artist_thumb'] or None, + } + ) + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 + except Exception as e: + logger.debug("Error creating album MBID mismatch finding for track %s: %s", + track['track_id'], e) + result.errors += 1 + def _create_mismatch_finding(self, context, result, track_id, title, artist_name, album_title, file_path, album_thumb, artist_thumb, mbid, mb_title, mb_artist, reason): @@ -373,7 +668,7 @@ class MbidMismatchDetectorJob(RepairJob): ) if context.create_finding: try: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='mbid_mismatch', severity='warning', @@ -400,7 +695,10 @@ class MbidMismatchDetectorJob(RepairJob): 'artist_thumb_url': artist_thumb or None, } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating MBID mismatch finding for track %s: %s", track_id, e) result.errors += 1 diff --git a/core/repair_jobs/metadata_gap_filler.py b/core/repair_jobs/metadata_gap_filler.py index ead800b1..51389336 100644 --- a/core/repair_jobs/metadata_gap_filler.py +++ b/core/repair_jobs/metadata_gap_filler.py @@ -177,7 +177,7 @@ class MetadataGapFillerJob(RepairJob): if context.create_finding: try: field_names = ', '.join(found_fields.keys()) - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='metadata_gap', severity='info', @@ -202,7 +202,10 @@ class MetadataGapFillerJob(RepairJob): 'artist_thumb_url': artist_thumb or None, } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating metadata gap finding for track %s: %s", track_id, e) result.errors += 1 diff --git a/core/repair_jobs/missing_cover_art.py b/core/repair_jobs/missing_cover_art.py index 1e406ee3..4c292873 100644 --- a/core/repair_jobs/missing_cover_art.py +++ b/core/repair_jobs/missing_cover_art.py @@ -138,7 +138,7 @@ class MissingCoverArtJob(RepairJob): # Create finding for user to approve if context.create_finding: try: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='missing_cover_art', severity='info', @@ -156,7 +156,10 @@ class MissingCoverArtJob(RepairJob): 'artist_thumb_url': artist_thumb or None, } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating cover art finding for album %s: %s", album_id, e) result.errors += 1 diff --git a/core/repair_jobs/orphan_file_detector.py b/core/repair_jobs/orphan_file_detector.py index b9f5371b..80c5a237 100644 --- a/core/repair_jobs/orphan_file_detector.py +++ b/core/repair_jobs/orphan_file_detector.py @@ -159,8 +159,8 @@ class OrphanFileDetectorJob(RepairJob): (first_artist and (clean_title, first_artist) in known_titles_clean) ): is_known = True - except Exception: - pass + except Exception as e: + logger.debug("tag-based orphan check: %s", e) # Last resort: parse title from filename pattern "NN - Title [Quality].ext" # and match against known titles. Catches files with unreadable tags. @@ -186,8 +186,8 @@ class OrphanFileDetectorJob(RepairJob): if clean_fn and (clean_fn, clean_fa) in known_titles_clean: is_known = True break - except Exception: - pass + except Exception as e: + logger.debug("filename-pattern orphan check: %s", e) if not is_known: orphan_files.append(fpath) @@ -217,7 +217,7 @@ class OrphanFileDetectorJob(RepairJob): stat = os.stat(fpath) ext = os.path.splitext(fpath)[1].lower().lstrip('.') if context.create_finding: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='orphan_file', severity='warning' if mass_orphan else 'info', @@ -241,7 +241,10 @@ class OrphanFileDetectorJob(RepairJob): 'mass_orphan': mass_orphan, } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating orphan finding for %s: %s", fpath, e) result.errors += 1 diff --git a/core/repair_jobs/single_album_dedup.py b/core/repair_jobs/single_album_dedup.py index cbcad948..a4ccdd5b 100644 --- a/core/repair_jobs/single_album_dedup.py +++ b/core/repair_jobs/single_album_dedup.py @@ -196,7 +196,7 @@ class SingleAlbumDedupJob(RepairJob): if context.create_finding: try: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='single_album_redundant', severity='info', @@ -235,7 +235,10 @@ class SingleAlbumDedupJob(RepairJob): 'artist_thumb_url': best_album_match.get('artist_thumb_url') or single.get('artist_thumb_url'), } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 except Exception as e: logger.debug("Error creating finding: %s", e) result.errors += 1 diff --git a/core/repair_jobs/track_number_repair.py b/core/repair_jobs/track_number_repair.py index 6246ac2c..8a569de6 100644 --- a/core/repair_jobs/track_number_repair.py +++ b/core/repair_jobs/track_number_repair.py @@ -237,7 +237,7 @@ class TrackNumberRepairJob(RepairJob): details['album_title'] = art_info['album_title'] if art_info.get('artist_name'): details['artist_name'] = art_info['artist_name'] - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='track_number_mismatch', severity='warning', @@ -248,7 +248,10 @@ class TrackNumberRepairJob(RepairJob): description=finding['description'], details=details ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 else: if _repair_single_track(fpath, fname, api_tracks, len(api_tracks), title_sim, context): result.auto_fixed += 1 diff --git a/core/repair_jobs/unknown_artist_fixer.py b/core/repair_jobs/unknown_artist_fixer.py index 6ceb42ab..bf037748 100644 --- a/core/repair_jobs/unknown_artist_fixer.py +++ b/core/repair_jobs/unknown_artist_fixer.py @@ -135,9 +135,14 @@ class UnknownArtistFixerJob(RepairJob): title = track['title'] or '' file_path = track['file_path'] - # Resolve actual file on disk - from core.repair_worker import _resolve_file_path - resolved = _resolve_file_path(file_path, transfer) + # Resolve actual file on disk via the shared library resolver + # (picks up library.music_paths + Plex library locations). + from core.library.path_resolver import resolve_library_file_path + resolved = resolve_library_file_path( + file_path, + transfer_folder=transfer, + config_manager=context.config_manager, + ) if not resolved or not os.path.exists(resolved): result.skipped += 1 continue @@ -185,7 +190,7 @@ class UnknownArtistFixerJob(RepairJob): desc_parts.append(f'Path: → {expected_rel}') if context.create_finding: - context.create_finding( + inserted = context.create_finding( job_id=self.job_id, finding_type='unknown_artist', severity='warning', @@ -212,7 +217,10 @@ class UnknownArtistFixerJob(RepairJob): 'cover_url': corrected.get('image_url', ''), } ) - result.findings_created += 1 + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 else: # Live mode — apply fix try: @@ -488,8 +496,8 @@ class UnknownArtistFixerJob(RepairJob): if not os.path.exists(sidecar_dst): try: shutil.move(sidecar_src, sidecar_dst) - except Exception: - pass + except Exception as e: + logger.debug("move sidecar file: %s", e) # Also move cover.jpg from old album folder cover_src = os.path.join(src_dir, 'cover.jpg') @@ -497,8 +505,8 @@ class UnknownArtistFixerJob(RepairJob): if os.path.isfile(cover_src) and not os.path.exists(cover_dst): try: shutil.copy2(cover_src, cover_dst) - except Exception: - pass + except Exception as e: + logger.debug("copy cover.jpg: %s", e) # Clean up empty directories parent = os.path.dirname(current_norm) diff --git a/core/repair_worker.py b/core/repair_worker.py index 799be10e..50d90b27 100644 --- a/core/repair_worker.py +++ b/core/repair_worker.py @@ -24,6 +24,7 @@ from core.metadata_service import ( get_source_priority, get_primary_source, ) +from core.library.path_resolver import resolve_library_file_path from core.repair_jobs import get_all_jobs from core.repair_jobs.base import JobContext, JobResult, RepairJob from utils.logging_config import get_logger @@ -33,29 +34,26 @@ logger = get_logger("repair_worker") AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'} -def _resolve_file_path(file_path, transfer_folder, download_folder=None): +def _resolve_file_path(file_path, transfer_folder, download_folder=None, + config_manager=None, plex_client=None): """Resolve a stored DB path to an actual file on disk. - Tries the raw path first, then progressively shorter suffixes against - configured directories. Handles cross-environment path mismatches - (e.g. Docker paths vs native Windows paths). + Thin wrapper around ``core.library.path_resolver.resolve_library_file_path`` + that preserves the legacy signature used by every caller in this module + and the repair-job modules. The shared resolver also probes the + user-configured ``library.music_paths`` and Plex-reported library + locations — which is what fixes the Album Completeness Auto-Fill + failure on Docker setups (issue #476). Pre-existing call sites that + don't pass ``config_manager`` keep the old transfer+download-only + behavior; sites that pass it in pick up the wider search automatically. """ - if not file_path: - return None - if os.path.exists(file_path): - return file_path - - path_parts = file_path.replace('\\', '/').split('/') - - for base_dir in [transfer_folder, download_folder]: - if not base_dir or not os.path.isdir(base_dir): - continue - for i in range(1, len(path_parts)): - candidate = os.path.join(base_dir, *path_parts[i:]) - if os.path.exists(candidate): - return candidate - - return None + return resolve_library_file_path( + file_path, + transfer_folder=transfer_folder, + download_folder=download_folder, + config_manager=config_manager, + plex_client=plex_client, + ) class RepairWorker: @@ -259,8 +257,22 @@ class RepairWorker: self._config_manager.set(f'repair.jobs.{job_id}.settings', current) def get_all_job_info(self) -> List[dict]: - """Get info for all jobs (for API response).""" + """Get info for all jobs (for API response). + + Includes ``pending_findings_count`` per job so the job-card + badge can show CURRENT pending state instead of the + ``last_run.findings_created`` historical scan count. Without + this, a scan that creates 372 findings + a subsequent bulk- + fix that resolves all of them leaves the badge displaying + "372 findings" while the Findings tab Pending filter shows 0 + — confusing UX flagged on the Library Maintenance page. + """ self._ensure_jobs_loaded() + + # Single query → per-job pending count dict. O(1) lookup per + # job instead of N round trips. + pending_by_job = self._get_pending_count_by_job() + jobs_info = [] for job_id, job in self._jobs.items(): config = self.get_job_config(job_id) @@ -286,9 +298,30 @@ class RepairWorker: 'last_run': last_run, 'next_run': next_run, 'is_running': self._current_job_id == job_id, + 'pending_findings_count': pending_by_job.get(job_id, 0), }) return jobs_info + def _get_pending_count_by_job(self) -> dict: + """Return ``{job_id: pending_count}`` for every job that has + any pending findings. Single SQL aggregation.""" + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT job_id, COUNT(*) FROM repair_findings + WHERE status = 'pending' + GROUP BY job_id + """) + return {row[0]: row[1] for row in cursor.fetchall()} + except Exception as e: + logger.debug("Error counting pending findings per job: %s", e) + return {} + finally: + if conn: + conn.close() + # ------------------------------------------------------------------ # Lifecycle # ------------------------------------------------------------------ @@ -546,8 +579,8 @@ class RepairWorker: if self._on_job_start: try: self._on_job_start(job_id, job.display_name) - except Exception: - pass + except Exception as e: + logger.debug("on_job_start callback failed: %s", e) # Record job start run_id = self._record_job_start(job_id) @@ -557,8 +590,8 @@ class RepairWorker: if self._on_job_progress: try: self._on_job_progress(job_id, **kwargs) - except Exception: - pass + except Exception as e: + logger.debug("on_job_progress callback failed: %s", e) # Build context context = JobContext( @@ -603,8 +636,8 @@ class RepairWorker: try: status = 'error' if result.errors > 0 and result.auto_fixed == 0 else 'finished' self._on_job_finish(job_id, status, result) - except Exception: - pass + except Exception as e: + logger.debug("on_job_finish callback failed: %s", e) logger.info( "Job %s complete: scanned=%d fixed=%d findings=%d errors=%d (%.1fs)", @@ -657,8 +690,18 @@ class RepairWorker: # ------------------------------------------------------------------ def _create_finding(self, job_id: str, finding_type: str, severity: str, entity_type: str, entity_id: str, file_path: str, - title: str, description: str, details: dict = None): - """Create a repair finding in the database.""" + title: str, description: str, details: dict = None) -> bool: + """Create a repair finding in the database. + + Returns: + True — a NEW pending row was inserted. + False — dedup-skipped (an equivalent row already exists with + status pending/resolved/dismissed) OR a DB error + occurred. Callers should only increment their + ``findings_created`` counter when this returns True + so the badge / scan log reports REAL new findings, + not silently-skipped duplicates. + """ conn = None try: conn = self.db._get_connection() @@ -674,7 +717,7 @@ class RepairWorker: """, (job_id, finding_type, entity_type, entity_id, file_path)) if cursor.fetchone(): - return # Already exists or was already fixed + return False # Already exists or was already fixed cursor.execute(""" INSERT INTO repair_findings @@ -687,8 +730,10 @@ class RepairWorker: json.dumps(details) if details else '{}' )) conn.commit() + return True except Exception as e: logger.debug("Error creating finding: %s", e) + return False finally: if conn: conn.close() @@ -843,6 +888,7 @@ class RepairWorker: 'duplicate_tracks': self._fix_duplicates, 'single_album_redundant': self._fix_single_album_redundant, 'mbid_mismatch': self._fix_mbid_mismatch, + 'album_mbid_mismatch': self._fix_album_mbid_mismatch, 'album_tag_inconsistency': self._fix_album_tag_inconsistency, 'incomplete_album': self._fix_incomplete_album, 'path_mismatch': self._fix_path_mismatch, @@ -1022,7 +1068,7 @@ class RepairWorker: download_folder = None if self._config_manager: download_folder = self._config_manager.get('soulseek.download_path', '') - resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder) or file_path + resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder, config_manager=self._config_manager) or file_path if not os.path.exists(resolved): return {'success': True, 'action': 'already_gone', @@ -1100,7 +1146,7 @@ class RepairWorker: download_folder = None if self._config_manager: download_folder = self._config_manager.get('soulseek.download_path', '') - resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder) or file_path + resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder, config_manager=self._config_manager) or file_path if not os.path.isfile(resolved): return {'success': False, 'error': f'File not found: {os.path.basename(file_path)}'} @@ -1118,8 +1164,8 @@ class RepairWorker: audio = MutagenFile(resolved) if audio: _, total_tracks = _read_track_number_tag(audio) - except Exception: - pass + except Exception as e: + logger.debug("Failed to read total_tracks tag from file: %s", e) total_tracks = int(total_tracks or 0) _fix_track_number_tag(resolved, int(correct_num), total_tracks) @@ -1139,8 +1185,8 @@ class RepairWorker: cursor.execute("UPDATE tracks SET file_path = ? WHERE file_path = ?", (new_path, resolved)) conn.commit() - except Exception: - pass + except Exception as e: + logger.debug("Failed to update DB file_path after rename: %s", e) finally: if conn: conn.close() @@ -1285,7 +1331,7 @@ class RepairWorker: files_deleted = 0 for fpath in remove_paths: try: - resolved = _resolve_file_path(fpath, self.transfer_folder, download_folder) + resolved = _resolve_file_path(fpath, self.transfer_folder, download_folder, config_manager=self._config_manager) if resolved and os.path.exists(resolved): os.remove(resolved) files_deleted += 1 @@ -1346,7 +1392,7 @@ class RepairWorker: if self._config_manager: download_folder = self._config_manager.get('soulseek.download_path', '') try: - resolved = _resolve_file_path(single_path, self.transfer_folder, download_folder) + resolved = _resolve_file_path(single_path, self.transfer_folder, download_folder, config_manager=self._config_manager) if resolved and os.path.exists(resolved): os.remove(resolved) file_deleted = True @@ -1414,7 +1460,7 @@ class RepairWorker: if self._config_manager: download_folder = self._config_manager.get('soulseek.download_path', '') try: - resolved = _resolve_file_path(track_path, self.transfer_folder, download_folder) + resolved = _resolve_file_path(track_path, self.transfer_folder, download_folder, config_manager=self._config_manager) if resolved and os.path.exists(resolved): os.remove(resolved) file_deleted = True @@ -1453,7 +1499,7 @@ class RepairWorker: # Resolve file download_folder = self._config_manager.get('soulseek.download_path', '') if self._config_manager else '' - resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder) if file_path else None + resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder, config_manager=self._config_manager) if file_path else None if not resolved or not os.path.exists(resolved): return {'success': False, 'error': f'File not found: {file_path}'} @@ -1498,8 +1544,8 @@ class RepairWorker: if not os.path.exists(d): try: shutil.move(s, d) - except Exception: - pass + except Exception as e: + logger.debug("Failed to move sidecar %s: %s", s, e) # Clean up empty dirs self._cleanup_empty_parents(resolved) @@ -1560,7 +1606,7 @@ class RepairWorker: if fix_action == 'delete': # Delete file + DB record if file_path: - resolved = _resolve_file_path(file_path, self.transfer_folder) + resolved = _resolve_file_path(file_path, self.transfer_folder, config_manager=self._config_manager) if resolved and os.path.exists(resolved): try: os.remove(resolved) @@ -1602,12 +1648,12 @@ class RepairWorker: logger.warning("Could not add to wishlist: %s", e) # Delete wrong file if file_path: - resolved = _resolve_file_path(file_path, self.transfer_folder) + resolved = _resolve_file_path(file_path, self.transfer_folder, config_manager=self._config_manager) if resolved and os.path.exists(resolved): try: os.remove(resolved) - except Exception: - pass + except Exception as e: + logger.debug("Failed to remove wrong file %s: %s", resolved, e) if track_id: try: conn = self.db._get_connection() @@ -1615,8 +1661,8 @@ class RepairWorker: cursor.execute("DELETE FROM tracks WHERE id = ?", (track_id,)) conn.commit() conn.close() - except Exception: - pass + except Exception as e: + logger.debug("Failed to delete wrong track row from DB: %s", e) return {'success': True, 'action': 'redownload', 'message': f'Added "{expected_title}" to wishlist, removed wrong file'} @@ -1661,7 +1707,7 @@ class RepairWorker: # Write corrected tags to the actual audio file if file_path: - resolved = _resolve_file_path(file_path, self.transfer_folder) + resolved = _resolve_file_path(file_path, self.transfer_folder, config_manager=self._config_manager) if resolved and os.path.exists(resolved): try: from core.tag_writer import write_tags_to_file @@ -1685,7 +1731,7 @@ class RepairWorker: download_folder = None if self._config_manager: download_folder = self._config_manager.get('soulseek.download_path', '') - resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder) + resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder, config_manager=self._config_manager) if not resolved or not os.path.exists(resolved): return {'success': False, 'error': f'File not found: {file_path}'} @@ -1706,6 +1752,47 @@ class RepairWorker: except Exception as e: return {'success': False, 'error': f'Failed to remove MBID: {str(e)}'} + def _fix_album_mbid_mismatch(self, entity_type, entity_id, file_path, details): + """Rewrite the dissenting track's album MBID to match the consensus. + + The detector flagged this track because its embedded + MUSICBRAINZ_ALBUMID disagreed with the consensus across the + album's other tracks. Fix is to rewrite the dissenter's tag — + does NOT touch the other tracks (they're already in agreement). + """ + consensus_mbid = details.get('consensus_mbid') + if not consensus_mbid: + return {'success': False, 'error': 'No consensus MBID in finding details'} + if not file_path: + return {'success': False, 'error': 'No file path associated with this finding'} + + download_folder = None + if self._config_manager: + download_folder = self._config_manager.get('soulseek.download_path', '') + resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder, + config_manager=self._config_manager) + if not resolved or not os.path.exists(resolved): + return {'success': False, 'error': f'File not found: {file_path}'} + + try: + from core.repair_jobs.mbid_mismatch_detector import _write_album_mbid_to_file + ok = _write_album_mbid_to_file(resolved, consensus_mbid) + if ok: + wrong = (details.get('wrong_mbid') or '')[:8] + consensus_short = consensus_mbid[:8] + title = details.get('title', 'track') + return { + 'success': True, + 'action': 'rewrote_album_mbid', + 'message': ( + f'Updated album MBID on "{title}" ' + f'({wrong}… → {consensus_short}…)' + ), + } + return {'success': False, 'error': 'Could not write album MBID — unsupported format or write failed'} + except Exception as e: + return {'success': False, 'error': f'Failed to rewrite album MBID: {str(e)}'} + def _fix_album_tag_inconsistency(self, entity_type, entity_id, file_path, details): """Normalize inconsistent tags across all tracks in an album to the canonical (majority) value.""" inconsistencies = details.get('inconsistencies', []) @@ -1731,7 +1818,7 @@ class RepairWorker: download_folder = None if self._config_manager: download_folder = self._config_manager.get('soulseek.download_path', '') - resolved = _resolve_file_path(track_file, self.transfer_folder, download_folder) + resolved = _resolve_file_path(track_file, self.transfer_folder, download_folder, config_manager=self._config_manager) if not resolved or not os.path.exists(resolved): continue @@ -1877,7 +1964,7 @@ class RepairWorker: album_folder = None for t in existing_tracks: - resolved = _resolve_file_path(t.file_path, self.transfer_folder, download_folder) + resolved = _resolve_file_path(t.file_path, self.transfer_folder, download_folder, config_manager=self._config_manager) if resolved and os.path.exists(resolved): album_folder = os.path.dirname(resolved) break @@ -1888,7 +1975,7 @@ class RepairWorker: # Detect filename pattern resolved_paths = [] for t in existing_tracks: - rp = _resolve_file_path(t.file_path, self.transfer_folder, download_folder) + rp = _resolve_file_path(t.file_path, self.transfer_folder, download_folder, config_manager=self._config_manager) if rp: resolved_paths.append(rp) filename_pattern = self._detect_filename_pattern(resolved_paths) @@ -2145,7 +2232,7 @@ class RepairWorker: """Move or copy a candidate track into the album folder and update DB.""" try: # Resolve source file - src_path = _resolve_file_path(candidate.file_path, self.transfer_folder, download_folder) + src_path = _resolve_file_path(candidate.file_path, self.transfer_folder, download_folder, config_manager=self._config_manager) if not src_path or not os.path.exists(src_path): return {'success': False, 'error': f'Source file not found: {candidate.file_path}'} @@ -2302,8 +2389,8 @@ class RepairWorker: album_thumb = album_row[2] album_track_count = album_row[3] spotify_album_id = album_row[4] if len(album_row) > 4 else None - except Exception: - pass + except Exception as e: + logger.debug("Failed to load album metadata for retag: %s", e) finally: if conn_meta: conn_meta.close() @@ -2442,8 +2529,8 @@ class RepairWorker: if not os.path.exists(sidecar_dst): try: shutil.move(sidecar_src, sidecar_dst) - except Exception: - pass + except Exception as e: + logger.debug("Failed to move sidecar %s: %s", sidecar_src, e) # Update DB file path conn = None @@ -2463,8 +2550,8 @@ class RepairWorker: cursor.execute( "UPDATE tracks SET file_path = ? WHERE file_path LIKE ? ESCAPE '^'", (dst, '%/' + escaped)) - except Exception: - pass + except Exception as e: + logger.debug("Suffix-match DB path update failed: %s", e) conn.commit() except Exception as e: logger.debug("DB path update failed for %s: %s", src, e) @@ -2534,7 +2621,7 @@ class RepairWorker: download_folder = None if self._config_manager: download_folder = self._config_manager.get('soulseek.download_path', '') - resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder) or file_path + resolved = _resolve_file_path(file_path, self.transfer_folder, download_folder, config_manager=self._config_manager) or file_path if not os.path.exists(resolved): return {'success': False, 'error': f'Source file not found: {file_path}'} @@ -2559,8 +2646,8 @@ class RepairWorker: if os.path.exists(out_path): try: os.remove(out_path) - except Exception: - pass + except Exception as e: + logger.debug("Failed to remove out_path after ffmpeg failure: %s", e) return {'success': False, 'error': f'ffmpeg conversion failed: {proc.stderr[:200] if proc.stderr else "unknown error"}'} # Update QUALITY tag @@ -2577,8 +2664,8 @@ class RepairWorker: from mutagen.mp4 import MP4FreeForm audio['----:com.apple.iTunes:QUALITY'] = [MP4FreeForm(quality_label.encode('utf-8'))] audio.save() - except Exception: - pass + except Exception as e: + logger.debug("Failed to write QUALITY tag on lossy copy: %s", e) # Embed cover art from source FLAC if codec in ('opus', 'aac'): @@ -2610,8 +2697,8 @@ class RepairWorker: fmt = MP4Cover.FORMAT_JPEG if 'jpeg' in pic.mime else MP4Cover.FORMAT_PNG dest_audio['covr'] = [MP4Cover(pic.data, imageformat=fmt)] dest_audio.save() - except Exception: - pass + except Exception as e: + logger.debug("Failed to embed cover art in lossy copy: %s", e) # Blasphemy Mode — uses the job's own setting, not the global lossy_copy one delete_original = False @@ -2637,8 +2724,8 @@ class RepairWorker: ) conn.commit() conn.close() - except Exception: - pass + except Exception as e: + logger.debug("Failed to update DB path after lossy conversion: %s", e) return {'success': True, 'action': 'converted_and_deleted', 'message': f'Converted to {quality_label} and deleted original'} except Exception as e: @@ -2651,8 +2738,8 @@ class RepairWorker: if os.path.exists(out_path): try: os.remove(out_path) - except Exception: - pass + except Exception as e: + logger.debug("Failed to remove out_path after timeout: %s", e) return {'success': False, 'error': 'Conversion timed out (120s)'} except Exception as e: return {'success': False, 'error': f'Conversion error: {e}'} @@ -2694,6 +2781,7 @@ class RepairWorker: fixable_types = ('dead_file', 'orphan_file', 'track_number_mismatch', 'missing_cover_art', 'metadata_gap', 'duplicate_tracks', 'single_album_redundant', 'mbid_mismatch', + 'album_mbid_mismatch', 'album_tag_inconsistency', 'incomplete_album', 'path_mismatch', 'missing_lossy_copy', diff --git a/core/replaygain.py b/core/replaygain.py index 4870f741..22e25c17 100644 --- a/core/replaygain.py +++ b/core/replaygain.py @@ -7,10 +7,13 @@ Tag writing uses mutagen directly to stay consistent with the rest of the codeba Supported formats: MP3, FLAC, OGG Vorbis, Opus, M4A/MP4 """ +import logging import re import subprocess from typing import Optional, Tuple, Dict +logger = logging.getLogger(__name__) + # ReplayGain 2.0 reference level (EBU R128) RG_REFERENCE_LUFS = -18.0 @@ -75,18 +78,48 @@ def analyze_track(file_path: str) -> Tuple[float, float]: stderr = result.stderr - # Parse integrated loudness: " I: -18.3 LUFS" - lufs_match = re.search(r'I:\s+([-\d.]+)\s+LUFS', stderr) - # Parse true peak: " Peak:\s+([-\d.]+) dBFS" (may appear per-channel; take max) - peak_matches = re.findall(r'Peak:\s+([-\d.]+)\s+dBFS', stderr) + # ebur128 emits two kinds of output: + # (a) Per-window progress lines like + # "[Parsed_ebur128_0 @ ...] t: 0.5 ... I: -70.0 LUFS ..." + # — the "I:" here is the PARTIAL integrated loudness up to that + # window. The very first window of nearly any track is silent + # (intro fade-in / encoder padding) and reads ~-70 LUFS. + # (b) A final "Summary:" block like: + # Summary: + # Integrated loudness: + # I: -14.3 LUFS + # ... + # True peak: + # Peak: -0.4 dBFS + # + # The OLD code used ``re.search('I:\s+...')`` which returned the FIRST + # match — i.e. the first per-window partial reading. For nearly every + # track that came back as ~-70 LUFS, producing gain = -18 - (-70) = + # +52.00 dB on EVERY track. (User report: "all tracks seem to get + # the same track_gain ... +52.00 dB".) + # + # Fix: anchor parsing to the Summary block so we always read the + # final integrated value, never a per-window partial. + summary_idx = stderr.rfind('Summary:') + if summary_idx >= 0: + summary_block = stderr[summary_idx:] + lufs_values = re.findall(r'I:\s+([-\d.]+)\s+LUFS', summary_block) + peak_matches = re.findall(r'Peak:\s+([-\d.]+)\s+dBFS', summary_block) + else: + # No Summary block in the output (truncated / unexpected ffmpeg + # version). Defensive fallback to the LAST per-window reading, + # which is at least closer to the final integrated value than + # the first window (which is always silence). + lufs_values = re.findall(r'I:\s+([-\d.]+)\s+LUFS', stderr) + peak_matches = re.findall(r'Peak:\s+([-\d.]+)\s+dBFS', stderr) - if not lufs_match: + if not lufs_values: raise RuntimeError( f"Could not parse ebur128 output for '{file_path}'. " f"FFmpeg exit code: {result.returncode}" ) - integrated_lufs = float(lufs_match.group(1)) + integrated_lufs = float(lufs_values[-1]) if peak_matches: true_peak_dbfs = max(float(v) for v in peak_matches) @@ -159,8 +192,8 @@ def read_replaygain_tags(file_path: str) -> Dict[str, Optional[str]]: result['track_peak'] = _mp4_rg(audio, _TAG_TRACK_PEAK) result['album_gain'] = _mp4_rg(audio, _TAG_ALBUM_GAIN) result['album_peak'] = _mp4_rg(audio, _TAG_ALBUM_PEAK) - except Exception: - pass + except Exception as e: + logger.debug("read replaygain tags failed: %s", e) return result @@ -177,8 +210,8 @@ def _read_id3_txxx(audio, description: str) -> Optional[str]: if frame_key.upper() == key.upper(): frame = audio.tags[frame_key] return str(frame.text[0]) if frame.text else None - except Exception: - pass + except Exception as e: + logger.debug("read id3 txxx frame failed: %s", e) return None @@ -188,8 +221,8 @@ def _vorbis_first(audio, key: str) -> Optional[str]: vals = audio.get(key) or audio.get(key.upper()) if vals: return str(vals[0]) - except Exception: - pass + except Exception as e: + logger.debug("read vorbis comment failed: %s", e) return None @@ -204,8 +237,8 @@ def _mp4_rg(audio, tag_name: str) -> Optional[str]: if hasattr(val, 'decode'): return val.decode('utf-8') return str(val) - except Exception: - pass + except Exception as e: + logger.debug("read mp4 replaygain atom failed: %s", e) return None diff --git a/core/runtime_state.py b/core/runtime_state.py index f952aee1..dfa11798 100644 --- a/core/runtime_state.py +++ b/core/runtime_state.py @@ -2,11 +2,14 @@ from __future__ import annotations +import logging import threading import time from functools import wraps from typing import Any, Dict, Optional +logger = logging.getLogger(__name__) + matched_context_lock = threading.Lock() matched_downloads_context: Dict[str, Dict[str, Any]] = {} tasks_lock = threading.Lock() @@ -57,8 +60,8 @@ def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True): if show_toast and _activity_toast_emitter is not None: try: _activity_toast_emitter("dashboard:toast", activity_item) - except Exception: - pass + except Exception as e: + logger.debug("emit activity toast failed: %s", e) return activity_item diff --git a/core/search/basic.py b/core/search/basic.py index 75ddd575..7c70eb29 100644 --- a/core/search/basic.py +++ b/core/search/basic.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) def run_basic_soulseek_search( query: str, - soulseek_client, + download_orchestrator, run_async: Callable, ) -> list[dict]: """Search Soulseek for `query`, normalize albums + tracks to one sorted list. @@ -22,7 +22,7 @@ def run_basic_soulseek_search( Returns dicts with `result_type` set to "album" or "track" and sorted by `quality_score` descending. Empty list on any failure (caller logs). """ - tracks, albums = run_async(soulseek_client.search(query)) + tracks, albums = run_async(download_orchestrator.search(query)) processed_albums = [] for album in albums: diff --git a/core/search/orchestrator.py b/core/search/orchestrator.py index 1c2cb4c1..da934759 100644 --- a/core/search/orchestrator.py +++ b/core/search/orchestrator.py @@ -49,7 +49,7 @@ class SearchDeps: spotify_client: Any hydrabase_client: Any hydrabase_worker: Any - soulseek_client: Any + download_orchestrator: Any fix_artist_image_url: Callable[[Optional[str]], Optional[str]] is_hydrabase_active: Callable[[], bool] get_metadata_fallback_source: Callable[[], str] @@ -289,10 +289,11 @@ def run_enhanced_search(query: str, requested_source: str, deps: SearchDeps) -> # --------------------------------------------------------------------------- def resolve_youtube_videos_client(deps: SearchDeps): - """Return the soulseek_client.youtube subclient or None when unavailable.""" - if not deps.soulseek_client: + """Return the YouTube download client (used for music-video search) + via the orchestrator's generic accessor, or None when unavailable.""" + if not deps.download_orchestrator or not hasattr(deps.download_orchestrator, 'client'): return None - return getattr(deps.soulseek_client, 'youtube', None) + return deps.download_orchestrator.client('youtube') def stream_youtube_videos(query: str, youtube_client, run_async: Callable) -> Iterator[str]: diff --git a/core/search/stream.py b/core/search/stream.py index cae298f3..c8af3f65 100644 --- a/core/search/stream.py +++ b/core/search/stream.py @@ -96,7 +96,7 @@ def stream_search_track( album_name: Optional[str], duration_ms: int, config_manager, - soulseek_client, + download_orchestrator, matching_engine, run_async: Callable, ) -> Optional[dict]: @@ -118,15 +118,9 @@ def stream_search_track( queries = _build_stream_queries(track_name, artist_name, effective_mode) - stream_clients = { - 'youtube': soulseek_client.youtube, - 'tidal': soulseek_client.tidal, - 'qobuz': soulseek_client.qobuz, - 'hifi': soulseek_client.hifi, - 'deezer_dl': soulseek_client.deezer_dl, - 'lidarr': soulseek_client.lidarr, - } - stream_client = stream_clients.get(effective_mode) + # Map mode name to canonical registry name (legacy 'deezer_dl' + # alias resolves to 'deezer' via the orchestrator's registry). + stream_client = download_orchestrator.client(effective_mode) use_direct_client = stream_client is not None max_peer_queue = config_manager.get('soulseek.max_peer_queue', 0) or 0 @@ -137,7 +131,7 @@ def stream_search_track( if use_direct_client: tracks_result, _ = run_async(stream_client.search(query, timeout=15)) else: - tracks_result, _ = run_async(soulseek_client.search(query, timeout=15)) + tracks_result, _ = run_async(download_orchestrator.search(query, timeout=15)) if not tracks_result: logger.info(f"No results for query '{query}', trying next...") diff --git a/core/seasonal_discovery.py b/core/seasonal_discovery.py index 0e7dfadc..159776f7 100644 --- a/core/seasonal_discovery.py +++ b/core/seasonal_discovery.py @@ -167,8 +167,8 @@ class SeasonalDiscoveryService: try: cursor.execute(f"ALTER TABLE {table} ADD COLUMN source TEXT NOT NULL DEFAULT 'spotify'") conn.commit() - except Exception: - pass # Column already exists + except Exception as e: + logger.debug("source column migration %s: %s", table, e) logger.info("Seasonal discovery database schema initialized") @@ -201,8 +201,8 @@ class SeasonalDiscoveryService: val = row[0] if isinstance(row, tuple) else row['value'] if val in ('northern', 'southern'): return val - except Exception: - pass + except Exception as e: + logger.debug("read hemisphere metadata failed: %s", e) return 'northern' def get_current_season(self) -> Optional[str]: diff --git a/core/soulid_worker.py b/core/soulid_worker.py index 5dc011bb..aef60f6d 100644 --- a/core/soulid_worker.py +++ b/core/soulid_worker.py @@ -280,8 +280,8 @@ class SoulIDWorker: if conn: try: conn.rollback() - except Exception: - pass + except Exception as _e: + logger.debug("rollback failed: %s", _e) return 0 finally: if conn: @@ -533,8 +533,8 @@ class SoulIDWorker: if conn: try: conn.rollback() - except Exception: - pass + except Exception as _e: + logger.debug("rollback failed: %s", _e) return 0 finally: if conn: @@ -595,8 +595,8 @@ class SoulIDWorker: if conn: try: conn.rollback() - except Exception: - pass + except Exception as _e: + logger.debug("rollback failed: %s", _e) return 0 finally: if conn: @@ -634,8 +634,8 @@ class SoulIDWorker: if conn: try: conn.rollback() - except Exception: - pass + except Exception as _e: + logger.debug("rollback failed: %s", _e) finally: if conn: conn.close() diff --git a/core/soulseek_client.py b/core/soulseek_client.py index 3110c305..1f7cb9c7 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -9,185 +9,46 @@ from pathlib import Path from utils.logging_config import get_logger from config.settings import config_manager from core.imports.filename import parse_filename_metadata +# Shared download-result dataclasses + plugin contract live in the +# neutral plugin package — every source uses the same types, so they +# belong there rather than this soulseek-specific module. +from core.download_plugins.types import ( + AlbumResult, + DownloadStatus, + SearchResult, + TrackResult, +) +from core.download_plugins.base import DownloadSourcePlugin logger = get_logger("soulseek_client") -@dataclass -class SearchResult: - """Base class for search results""" - username: str - filename: str - size: int - bitrate: Optional[int] - duration: Optional[int] # Duration in milliseconds (converted from slskd's seconds) - quality: str - free_upload_slots: int - upload_speed: int - queue_length: int - result_type: str = "track" # "track" or "album" - - @property - def quality_score(self) -> float: - quality_weights = { - 'flac': 1.0, - 'mp3': 0.8, - 'ogg': 0.7, - 'aac': 0.6, - 'wma': 0.5 - } - - base_score = quality_weights.get(self.quality.lower(), 0.3) - - if self.bitrate: - if self.bitrate >= 320: - base_score += 0.2 - elif self.bitrate >= 256: - base_score += 0.1 - elif self.bitrate < 128: - base_score -= 0.2 - - # Free upload slots - if self.free_upload_slots == 0: - base_score -= 0.15 - elif self.free_upload_slots > 0: - base_score += 0.05 - # Upload speed in bytes/sec (tiered) - if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps - base_score += 0.15 - elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps - base_score += 0.10 - elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps - base_score += 0.05 - elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps - base_score -= 0.05 +# slskd HTTP timeouts. Issue #499: long-running download sessions +# (~2-3hr) wedged because ``aiohttp.ClientSession()`` was constructed +# with no timeout — when slskd hung on a request (overloaded, network +# blip, internal stall), the HTTP call blocked indefinitely. The +# download worker thread blocked with it. Once the +# ``ThreadPoolExecutor(max_workers=3)`` had all 3 threads wedged, +# no further downloads could start and the user had to restart the +# container. +# +# Every slskd API call is metadata-level (search submission, status +# polls, download enqueue, transfer state queries) — none stream files. +# slskd handles file transfer via its own peer-to-peer infrastructure +# entirely outside our HTTP requests. So generous-but-bounded timeouts +# are safe and won't kill legitimate operations. +# +# Failures surface as caught exceptions in the existing +# ``except Exception`` blocks → logged + return None → caller treats +# as a normal failure (same as a 5xx response). No new error path. +_SLSKD_DEFAULT_TIMEOUT = aiohttp.ClientTimeout( + total=120, # hard ceiling — no single slskd call should take >2min + connect=15, # TCP connect to slskd + sock_read=60, # per-chunk read; slskd shouldn't go silent for >60s +) - # Queue length (graduated penalty) - if self.queue_length > 50: - base_score -= 0.25 - elif self.queue_length > 20: - base_score -= 0.15 - elif self.queue_length > 10: - base_score -= 0.10 - return min(base_score, 1.0) - -@dataclass -class TrackResult(SearchResult): - """Individual track search result""" - artist: Optional[str] = None - title: Optional[str] = None - album: Optional[str] = None - track_number: Optional[int] = None - - def __post_init__(self): - self.result_type = "track" - # Try to extract metadata from filename if not provided - if not self.title or not self.artist: - self._parse_filename_metadata() - - def _parse_filename_metadata(self): - """Extract artist, title, album from filename patterns""" - parsed = parse_filename_metadata(self.filename) - if not self.artist and parsed.get("artist"): - self.artist = parsed["artist"] - if not self.title and parsed.get("title"): - self.title = parsed["title"] - if not self.album and parsed.get("album"): - self.album = parsed["album"] - if self.track_number is None: - track_number = parsed.get("track_number") - if track_number is not None: - self.track_number = track_number - -@dataclass -class AlbumResult: - """Album/folder search result containing multiple tracks""" - username: str - album_path: str # Directory path - album_title: str - artist: Optional[str] - track_count: int - total_size: int - tracks: List[TrackResult] - dominant_quality: str # Most common quality in album - year: Optional[str] = None - free_upload_slots: int = 0 - upload_speed: int = 0 - queue_length: int = 0 - result_type: str = "album" - - @property - def quality_score(self) -> float: - """Calculate album quality score based on dominant quality and track count""" - quality_weights = { - 'flac': 1.0, - 'mp3': 0.8, - 'ogg': 0.7, - 'aac': 0.6, - 'wma': 0.5 - } - - base_score = quality_weights.get(self.dominant_quality.lower(), 0.3) - - # Bonus for complete albums (typically 8-15 tracks) - if 8 <= self.track_count <= 20: - base_score += 0.1 - elif self.track_count > 20: - base_score += 0.05 - - # Free upload slots - if self.free_upload_slots == 0: - base_score -= 0.15 - elif self.free_upload_slots > 0: - base_score += 0.05 - - # Upload speed in bytes/sec (tiered) - if self.upload_speed >= 5_000_000: # ~5 MB/s / 40 Mbps - base_score += 0.15 - elif self.upload_speed >= 1_000_000: # ~1 MB/s / 8 Mbps - base_score += 0.10 - elif self.upload_speed >= 500_000: # ~500 KB/s / 4 Mbps - base_score += 0.05 - elif self.upload_speed < 100_000: # ~100 KB/s / 800 kbps - base_score -= 0.05 - - # Queue length (graduated penalty) - if self.queue_length > 50: - base_score -= 0.25 - elif self.queue_length > 20: - base_score -= 0.15 - elif self.queue_length > 10: - base_score -= 0.10 - - return min(base_score, 1.0) - - @property - def size_mb(self) -> int: - """Album size in MB""" - return self.total_size // (1024 * 1024) - - @property - def average_track_size_mb(self) -> float: - """Average track size in MB""" - if self.track_count > 0: - return self.size_mb / self.track_count - return 0 - -@dataclass -class DownloadStatus: - id: str - filename: str - username: str - state: str - progress: float - size: int - transferred: int - speed: int - time_remaining: Optional[int] = None - file_path: Optional[str] = None - -class SoulseekClient: +class SoulseekClient(DownloadSourcePlugin): def __init__(self): self.base_url: Optional[str] = None self.api_key: Optional[str] = None @@ -283,25 +144,27 @@ class SoulseekClient: url = f"{self.base_url}/api/v0/{endpoint}" - # Create a fresh session for each thread/event loop to avoid conflicts + # Create a fresh session for each thread/event loop to avoid conflicts. + # Bounded timeout (issue #499) prevents the worker thread from + # wedging if slskd hangs. session = None try: - session = aiohttp.ClientSession() - + session = aiohttp.ClientSession(timeout=_SLSKD_DEFAULT_TIMEOUT) + headers = self._get_headers() - + if 'json' in kwargs: logger.debug(f"JSON payload: {kwargs['json']}") - + async with session.request( - method, - url, + method, + url, headers=headers, **kwargs ) as response: response_text = await response.text() - - + + if response.status in [200, 201, 204]: # Accept 200 OK, 201 Created, and 204 No Content self._last_401_logged = False # Reset on success try: @@ -320,7 +183,7 @@ class SoulseekClient: else: # Enhanced error logging for better debugging error_detail = response_text if response_text.strip() else "No error details provided" - + # Reduce noise for expected 404s (e.g. status checks for YouTube downloads) # and repeated 401s (slskd not running / bad credentials) if response.status == 404: @@ -334,9 +197,17 @@ class SoulseekClient: self._last_401_logged = False logger.error(f"API request failed: HTTP {response.status} ({response.reason}) - {error_detail}") logger.debug(f"Failed request: {method} {url}") - + return None - + + except asyncio.TimeoutError: + # Issue #499: explicit handling so the worker thread unblocks + # instead of staying wedged on the HTTP call. + logger.warning( + f"slskd request timed out after {_SLSKD_DEFAULT_TIMEOUT.total}s: " + f"{method} {url} — slskd may be overloaded or unreachable" + ) + return None except Exception as e: logger.error(f"Error making API request: {e}") return None @@ -345,36 +216,38 @@ class SoulseekClient: if session: try: await session.close() - except: - pass - + except Exception as _e: + logger.debug("aiohttp session close: %s", _e) + async def _make_direct_request(self, method: str, endpoint: str, **kwargs) -> Optional[Dict[str, Any]]: """Make a direct request to slskd without /api/v0/ prefix (for endpoints that work directly)""" if not self.base_url: logger.debug("Soulseek client not configured") return None - + url = f"{self.base_url}/{endpoint}" - - # Create a fresh session for each thread/event loop to avoid conflicts + + # Create a fresh session for each thread/event loop to avoid conflicts. + # Bounded timeout (issue #499) prevents the worker thread from + # wedging if slskd hangs. session = None try: - session = aiohttp.ClientSession() - + session = aiohttp.ClientSession(timeout=_SLSKD_DEFAULT_TIMEOUT) + headers = self._get_headers() - + if 'json' in kwargs: logger.debug(f"JSON payload: {kwargs['json']}") - + async with session.request( - method, - url, + method, + url, headers=headers, **kwargs ) as response: response_text = await response.text() - - + + if response.status == 200: try: return await response.json() @@ -385,7 +258,13 @@ class SoulseekClient: else: logger.error(f"Direct API request failed: {response.status} - {response_text}") return None - + + except asyncio.TimeoutError: + logger.warning( + f"slskd direct request timed out after {_SLSKD_DEFAULT_TIMEOUT.total}s: " + f"{method} {url} — slskd may be overloaded or unreachable" + ) + return None except Exception as e: logger.error(f"Error making direct API request: {e}") return None @@ -394,9 +273,9 @@ class SoulseekClient: if session: try: await session.close() - except: - pass - + except Exception as _e: + logger.debug("aiohttp direct session close: %s", _e) + def _process_search_responses(self, responses_data: List[Dict[str, Any]]) -> tuple[List[TrackResult], List[AlbumResult]]: """Process search response data into TrackResult and AlbumResult objects""" from collections import defaultdict @@ -1684,7 +1563,7 @@ class SoulseekClient: # Try to get Swagger/OpenAPI documentation swagger_url = f"{self.base_url}/swagger/v1/swagger.json" - session = aiohttp.ClientSession() + session = aiohttp.ClientSession(timeout=_SLSKD_DEFAULT_TIMEOUT) try: headers = self._get_headers() async with session.get(swagger_url, headers=headers) as response: @@ -1735,15 +1614,15 @@ class SoulseekClient: else: # Try different endpoints without /api/v0 prefix simple_url = f"{self.base_url}/{endpoint}" - session = aiohttp.ClientSession() + session = aiohttp.ClientSession(timeout=_SLSKD_DEFAULT_TIMEOUT) try: headers = self._get_headers() async with session.get(simple_url, headers=headers) as resp: if resp.status in [200, 405]: # 405 means endpoint exists but wrong method available_endpoints[f"direct_{endpoint}"] = f"Status: {resp.status}" logger.info(f"[OK] Direct endpoint available: {simple_url} (Status: {resp.status})") - except: - pass + except Exception as _e: + logger.debug("direct endpoint probe %s: %s", endpoint, _e) finally: await session.close() diff --git a/core/soulsync_client.py b/core/soulsync_client.py index 39653768..2c43d7d3 100644 --- a/core/soulsync_client.py +++ b/core/soulsync_client.py @@ -98,6 +98,14 @@ class SoulSyncTrack: self.path = file_path self.bitRate = tags['bitrate'] self.suffix = os.path.splitext(file_path)[1].lstrip('.').lower() + # File size in bytes (powers Library Disk Usage card on Stats). + # SoulSync standalone is the only "server" where we can read + # size from disk directly — Plex/Jellyfin/Navidrome get theirs + # from the API response. + try: + self.file_size = os.path.getsize(file_path) if os.path.exists(file_path) else None + except OSError: + self.file_size = None def artist(self): return self._artist_ref @@ -182,7 +190,10 @@ class SoulSyncArtist: return self._albums -class SoulSyncClient: +from core.media_server.contract import MediaServerClient + + +class SoulSyncClient(MediaServerClient): """Filesystem-based media server client for standalone SoulSync operation. Scans the Transfer folder recursively, reads audio file tags, and @@ -242,8 +253,8 @@ class SoulSyncClient: if self._progress_callback: try: self._progress_callback(msg) - except Exception: - pass + except Exception as e: + logger.debug("progress callback failed: %s", e) # ── Core Scanning ── diff --git a/core/soundcloud_client.py b/core/soundcloud_client.py new file mode 100644 index 00000000..530d6f72 --- /dev/null +++ b/core/soundcloud_client.py @@ -0,0 +1,604 @@ +""" +SoundCloud Download Client +Alternative music download source using yt-dlp's SoundCloud extractor. + +This client provides: +- SoundCloud search via the `scsearch` extractor +- Anonymous public-track downloads (no auth required) +- Drop-in replacement compatible with the existing TidalDownloadClient / + QobuzClient / HiFiClient / DeezerDownloadClient interface + +The client is intentionally NOT wired into web_server.py, settings UI, or +the unified search dispatch. Build/test in isolation first; integration +ships in a follow-up PR once the client is verified end-to-end. + +Quality reality check: +- Anonymous SoundCloud serves 128 kbps MP3 for most public tracks. A few + uploaders flag tracks for 256 kbps AAC streaming via SoundCloud Go+, but + those require an authenticated session; we only fetch the publicly + available transcoding. +- No FLAC. SoundCloud doesn't expose lossless to anyone, ever. +- Many tracks (especially DJ mixes) are >60 minutes long. Downloads can + be large; the integrity check still applies downstream. +""" + +import os +import re +import asyncio +import uuid +import time +from typing import List, Optional, Dict, Any, Tuple, Callable +from pathlib import Path + +try: + import yt_dlp +except ImportError: + yt_dlp = None + +from utils.logging_config import get_logger +from config.settings import config_manager + +# Standard data structures shared across all download clients so downstream +# matching/post-processing stays source-agnostic. +from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus + +logger = get_logger("soundcloud_client") + + +# Quality tiers — SoundCloud anonymous access only really delivers one +# quality, but we keep the structure consistent with other clients so +# UI/settings can reference a familiar shape later. +QUALITY_MAP = { + 'standard': { + 'label': 'MP3 128kbps', + 'extension': 'mp3', + 'bitrate': 128, + 'codec': 'mp3', + }, +} + +# Hard limit on yt-dlp result count per search to keep search latency bounded. +DEFAULT_SEARCH_LIMIT = 25 +MAX_SEARCH_LIMIT = 50 + +# Shorthand for `scsearch:` — yt-dlp's SoundCloud search prefix. +# Returns up to N tracks ranked by SoundCloud's own relevance. +_SC_SEARCH_PREFIX = "scsearch" + +# Minimum acceptable download size — anything below is almost certainly a +# broken response or a "preview snippet" file. Real SoundCloud audio for +# even a 1-minute track exceeds 100KB at 128kbps. +_MIN_AUDIO_SIZE_BYTES = 100 * 1024 + +# Filesystem-safe replacement for the platform's reserved characters. +_UNSAFE_FILENAME_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]') + + +def _sanitize_filename(name: str) -> str: + """Replace reserved filesystem characters with underscores.""" + cleaned = _UNSAFE_FILENAME_CHARS.sub('_', name) + # Collapse runs of underscores so we don't produce "track______name". + cleaned = re.sub(r'_{2,}', '_', cleaned).strip(' ._') + return cleaned or 'soundcloud_track' + + +from core.download_plugins.base import DownloadSourcePlugin + + +class SoundcloudClient(DownloadSourcePlugin): + """SoundCloud download client built on yt-dlp's SoundCloud extractor. + + Mirrors the public surface of TidalDownloadClient / QobuzClient so the + eventual integration step is a wiring change, not a refactor. + """ + + def __init__(self, download_path: Optional[str] = None): + if yt_dlp is None: + logger.warning("yt-dlp not installed — SoundCloud downloads unavailable") + + if download_path is None: + download_path = config_manager.get('soulseek.download_path', './downloads') + + self.download_path = Path(download_path) + self.download_path.mkdir(parents=True, exist_ok=True) + + logger.info(f"SoundCloud client using download path: {self.download_path}") + + # Optional shutdown predicate — wired by the runtime to short-circuit + # in-flight downloads when the worker is shutting down. + self.shutdown_check: Optional[Callable[[], bool]] = None + + self._engine = None + + # ------------------------------------------------------------------ + # Lifecycle / availability + # ------------------------------------------------------------------ + + def set_engine(self, engine): + """Engine callback — wires the central thread worker + state store.""" + self._engine = engine + + def set_shutdown_check(self, check_callable: Optional[Callable[[], bool]]) -> None: + self.shutdown_check = check_callable + + def is_available(self) -> bool: + """True when yt-dlp is installed. Anonymous SoundCloud needs no auth.""" + return yt_dlp is not None + + def is_configured(self) -> bool: + """True if the client has everything it needs to operate. + + Anonymous-only for now — if yt-dlp is present, we're configured. + Future tier-2 OAuth would gate on stored credentials here. + """ + return self.is_available() + + def is_authenticated(self) -> bool: + """Anonymous-only client — always False until OAuth tier ships.""" + return False + + async def check_connection(self) -> bool: + """Run a tiny SoundCloud query to verify the network path works.""" + if not self.is_available(): + return False + + try: + tracks, _albums = await self.search("test", timeout=15) + return True + except Exception as exc: + logger.warning(f"SoundCloud connection check failed: {exc}") + return False + + # ------------------------------------------------------------------ + # Search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback: Optional[Callable] = None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + """Search SoundCloud for the given query. + + Returns (tracks, albums). SoundCloud has no album concept (only + playlists, which don't map to the album model the rest of SoulSync + expects), so the album list is always empty. + """ + if not self.is_available(): + logger.warning("SoundCloud not available for search (yt-dlp missing)") + return ([], []) + + if not query or not isinstance(query, str): + logger.warning(f"Invalid SoundCloud search query: {query!r}") + return ([], []) + + # SoundCloud or a transient yt-dlp parse can fail; the caller still + # gets an empty list, never a raised exception. + limit = min(MAX_SEARCH_LIMIT, max(1, DEFAULT_SEARCH_LIMIT)) + search_url = f"{_SC_SEARCH_PREFIX}{limit}:{query}" + + logger.info(f"Searching SoundCloud for: {query} (limit={limit})") + + loop = asyncio.get_event_loop() + try: + entries = await loop.run_in_executor(None, self._extract_search_entries, search_url) + except Exception as exc: + logger.error(f"SoundCloud search failed: {exc}") + return ([], []) + + if not entries: + logger.info(f"No SoundCloud results for: {query}") + return ([], []) + + track_results: List[TrackResult] = [] + for entry in entries: + try: + converted = self._sc_to_track_result(entry) + if converted is not None: + track_results.append(converted) + except Exception as exc: + logger.debug(f"Skipping SoundCloud entry conversion error: {exc}") + + logger.info(f"Found {len(track_results)} SoundCloud tracks for '{query}'") + return (track_results, []) + + def _extract_search_entries(self, search_url: str) -> List[Dict[str, Any]]: + """Run yt-dlp in flat-extract mode to get a quick list of search hits. + + Flat extraction skips per-entry HTTP roundtrips during search, so + results come back in roughly the time of one SoundCloud API call. + Per-entry resolution happens later, at download time. + """ + opts = { + 'quiet': True, + 'no_warnings': True, + 'skip_download': True, + 'extract_flat': True, + 'noplaylist': False, + } + + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(search_url, download=False) + if not info or not isinstance(info, dict): + return [] + entries = info.get('entries') or [] + return [e for e in entries if isinstance(e, dict)] + + def _sc_to_track_result(self, entry: Dict[str, Any]) -> Optional[TrackResult]: + """Convert a yt-dlp SoundCloud entry into the standard TrackResult. + + Returns None when the entry lacks a usable URL — the worker can't + download it later anyway, so dropping it from search results saves + the user a confused "click → fail" interaction. + """ + url = entry.get('url') or entry.get('webpage_url') + if not url: + return None + + # yt-dlp's flat-extract entry has `id`, `title`, `uploader`, and + # sometimes `duration`. Other fields (artist, album) are usually + # only present after a full extraction. + title = (entry.get('title') or '').strip() + uploader = (entry.get('uploader') or entry.get('uploader_id') or '').strip() + + # Many SoundCloud titles are formatted "Artist - Title" by the + # uploader. If we don't have a separate artist field, try to peel + # one off the title; fall back to the uploader otherwise. + artist, parsed_title = self._split_artist_from_title(title, uploader) + + duration_seconds = entry.get('duration') + duration_ms: Optional[int] = None + if isinstance(duration_seconds, (int, float)) and duration_seconds > 0: + duration_ms = int(duration_seconds * 1000) + + sc_track_id = str(entry.get('id') or '') + if not sc_track_id: + # No stable id → can't pass through the filename-based dispatch. + return None + + display_name = f"{artist} - {parsed_title}".strip(' -') or parsed_title or sc_track_id + # ``filename`` is the dispatch key downstream code uses to identify + # the download. We cram the SoundCloud URL into it so the download + # worker has everything it needs without re-querying SoundCloud. + filename = f"{sc_track_id}||{url}||{display_name}" + + track_result = TrackResult( + username='soundcloud', + filename=filename, + size=0, + bitrate=128, # Anonymous SoundCloud cap + duration=duration_ms, + quality='mp3', + free_upload_slots=999, + upload_speed=999_999, + queue_length=0, + artist=artist or None, + title=parsed_title or None, + album=None, + track_number=None, + _source_metadata={ + 'source': 'soundcloud', + 'track_id': sc_track_id, + 'permalink_url': url, + 'uploader': uploader or None, + 'duration_seconds': duration_seconds, + }, + ) + return track_result + + @staticmethod + def _split_artist_from_title(title: str, uploader: str) -> Tuple[str, str]: + """Best-effort parse of "Artist - Title" out of a SoundCloud title. + + SoundCloud uploaders frequently format their tracks as + ``"Artist Name - Track Title"``. When that pattern is present, we + use it. Otherwise the uploader's display name is the artist and + the whole title stays as the title. + + This is best-effort — the matching logic downstream still has the + original title in `_source_metadata` and can fall back to fuzzy + comparison if our split was wrong. + """ + if not title: + return (uploader, '') + + # Match the FIRST " - " (most common separator). Avoid em-dash etc + # for now; uploaders use plain hyphen 95%+ of the time. + if ' - ' in title: + artist_part, _sep, title_part = title.partition(' - ') + artist_part = artist_part.strip() + title_part = title_part.strip() + # Sanity: very short artist parts (< 2 chars) are usually + # punctuation noise, not real names. + if len(artist_part) >= 2 and title_part: + return (artist_part, title_part) + + return (uploader, title) + + # ------------------------------------------------------------------ + # Download orchestration + # ------------------------------------------------------------------ + + async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]: + """Kick off a SoundCloud download via engine.worker.""" + parts = filename.split('||', 2) + if len(parts) < 2: + logger.error(f"Invalid SoundCloud filename format: {filename}") + return None + + sc_track_id = parts[0] + permalink_url = parts[1] + display_name = parts[2] if len(parts) > 2 else sc_track_id + + if not sc_track_id or not permalink_url: + logger.error(f"Missing SoundCloud track id or url in: {filename}") + return None + if self._engine is None: + # Raise rather than return None so the orchestrator's + # download_with_fallback surfaces a real warning + tries + # the next source. Returning None silently dropped the + # download with no user feedback (per JohnBaumb). + raise RuntimeError("SoundCloud client has no engine reference — cannot dispatch download") + + logger.info(f"Starting SoundCloud download: {display_name}") + + # Worker passes (download_id, target_id, display_name) to impl; + # SoundCloud's _download_sync wants permalink_url (not track_id), + # so adapt by closing over permalink_url here. + def _impl(download_id, _target_id, _display_name): + return self._download_sync(download_id, permalink_url, display_name) + + return self._engine.worker.dispatch( + source_name='soundcloud', + target_id=sc_track_id, + display_name=display_name, + original_filename=filename, + impl_callable=_impl, + extra_record_fields={ + 'track_id': sc_track_id, + 'permalink_url': permalink_url, + 'display_name': display_name, + }, + ) + + def _download_sync(self, download_id: str, permalink_url: str, + display_name: str) -> Optional[str]: + """Synchronously download a single SoundCloud track via yt-dlp. + + Returns the absolute path to the saved file, or None on failure. + Handles the shutdown_check via a per-progress yt-dlp hook so a + long DJ mix can still be interrupted mid-download. + """ + if not self.is_available(): + logger.error("SoundCloud download attempted with yt-dlp unavailable") + return None + + safe_name = _sanitize_filename(display_name) + # yt-dlp resolves the actual extension at download time (almost + # always .mp3 for anonymous SoundCloud). The %(ext)s placeholder + # lets it pick. + out_template = str(self.download_path / f"{safe_name}.%(ext)s") + speed_start = time.time() + + def _progress_hook(progress: Dict[str, Any]) -> None: + if self.shutdown_check and self.shutdown_check(): + # yt-dlp catches DownloadError and treats other exceptions + # as fatal — raise something it'll surface as a clean abort. + raise yt_dlp.utils.DownloadError("Shutdown requested") + + status = progress.get('status') + if status == 'downloading': + downloaded = int(progress.get('downloaded_bytes') or 0) + total = int(progress.get('total_bytes') or progress.get('total_bytes_estimate') or 0) + + # SoundCloud serves HLS-segmented audio. yt-dlp doesn't know + # the final byte total upfront — `total_bytes` and + # `total_bytes_estimate` reflect the CURRENT FRAGMENT size, + # not the whole download, so a byte-based percentage stays + # near 0 until the very end. Fall back to fragment progress + # which yt-dlp DOES populate accurately for HLS. + fragment_index = progress.get('fragment_index') + fragment_count = progress.get('fragment_count') + if (fragment_index is not None and fragment_count + and fragment_count > 0): + self._update_download_progress_fragmented( + download_id, downloaded, fragment_index, fragment_count, speed_start, + ) + else: + self._update_download_progress(download_id, downloaded, total, speed_start) + elif status == 'finished': + # yt-dlp signals 'finished' once the bytes are on disk; the + # final size is authoritative. Mark progress at 99% — the + # outer thread flips to 100% / Completed once we return. + downloaded = int(progress.get('total_bytes') or progress.get('downloaded_bytes') or 0) + self._update_download_progress(download_id, downloaded, downloaded, speed_start) + + opts = { + 'quiet': True, + 'no_warnings': True, + 'noplaylist': True, + 'outtmpl': out_template, + 'progress_hooks': [_progress_hook], + 'format': 'bestaudio/best', + # Disable yt-dlp's own retry storm — surface failures fast so + # the worker decides whether to retry from another source. + 'retries': 1, + 'fragment_retries': 1, + } + + try: + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(permalink_url, download=True) + except Exception as exc: + # Cover yt_dlp.utils.DownloadError + everything else. + logger.warning(f"SoundCloud download failed for '{display_name}': {exc}") + return None + + if not isinstance(info, dict): + logger.warning(f"SoundCloud yt-dlp returned no info dict for '{display_name}'") + return None + + # yt-dlp's prepare_filename gives us the resolved on-disk path + # honoring outtmpl + the actual extension it picked. + try: + with yt_dlp.YoutubeDL(opts) as ydl: + resolved_path = ydl.prepare_filename(info) + except Exception as exc: + logger.warning(f"Could not resolve final filename for '{display_name}': {exc}") + return None + + if not resolved_path or not os.path.exists(resolved_path): + logger.warning(f"SoundCloud download claimed success but file missing: {resolved_path}") + return None + + try: + final_size = os.path.getsize(resolved_path) + except OSError: + final_size = 0 + + if final_size < _MIN_AUDIO_SIZE_BYTES: + logger.warning( + f"SoundCloud download too small ({final_size} bytes) for " + f"'{display_name}' — likely a preview snippet, discarding" + ) + try: + os.remove(resolved_path) + except OSError: + pass + return None + + logger.info( + f"SoundCloud download complete: {resolved_path} " + f"({final_size / (1024 * 1024):.1f} MB)" + ) + return resolved_path + + def _update_download_progress_fragmented(self, download_id: str, downloaded: int, + fragment_index: int, fragment_count: int, + speed_start: float) -> None: + """HLS-aware progress update — fragment_index / fragment_count + gives an accurate signal even when each fragment's + ``total_bytes`` only describes the current fragment.""" + if self._engine is None: + return + record = self._engine.get_record('soundcloud', download_id) + if record is None: + return + + now = time.time() + elapsed = now - speed_start + speed = int(downloaded / elapsed) if elapsed > 0 else 0 + + progress = round(min((fragment_index / fragment_count) * 100, 99.9), 1) if fragment_count > 0 else 0.0 + + # Estimate total size from per-fragment average. + if fragment_index > 0 and downloaded > 0: + est_total = int(downloaded * (fragment_count / fragment_index)) + else: + est_total = downloaded + + time_remaining: Optional[int] = None + remaining_fragments = max(0, fragment_count - fragment_index) + if speed > 0 and remaining_fragments > 0 and fragment_index > 0: + seconds_per_fragment = elapsed / fragment_index if fragment_index > 0 else 0 + time_remaining = int(remaining_fragments * seconds_per_fragment) + + self._engine.update_record('soundcloud', download_id, { + 'transferred': downloaded, + 'speed': speed, + 'progress': progress, + 'size': est_total, + 'time_remaining': time_remaining, + }) + + def _update_download_progress(self, download_id: str, downloaded: int, + total: int, speed_start: float) -> None: + """Byte-based progress update for non-HLS streams.""" + if self._engine is None: + return + record = self._engine.get_record('soundcloud', download_id) + if record is None: + return + + now = time.time() + elapsed = now - speed_start + speed = int(downloaded / elapsed) if elapsed > 0 else 0 + + progress = record.get('progress', 0.0) + if total > 0: + progress = round(min((downloaded / total) * 100, 99.9), 1) + + time_remaining: Optional[int] = None + if speed > 0 and total > 0: + remaining = total - downloaded + if remaining > 0: + time_remaining = int(remaining / speed) + + self._engine.update_record('soundcloud', download_id, { + 'transferred': downloaded, + 'size': total, + 'speed': speed, + 'progress': progress, + 'time_remaining': time_remaining, + }) + + # ------------------------------------------------------------------ + # Status / cancellation + # ------------------------------------------------------------------ + + def _record_to_status(self, record: dict) -> DownloadStatus: + return DownloadStatus( + id=record['id'], + filename=record['filename'], + username=record['username'], + state=record['state'], + progress=record['progress'], + size=record.get('size', 0), + transferred=record.get('transferred', 0), + speed=record.get('speed', 0), + time_remaining=record.get('time_remaining'), + file_path=record.get('file_path'), + ) + + async def get_all_downloads(self) -> List[DownloadStatus]: + if self._engine is None: + return [] + return [ + self._record_to_status(record) + for record in self._engine.iter_records_for_source('soundcloud') + ] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + if self._engine is None: + return None + record = self._engine.get_record('soundcloud', download_id) + return self._record_to_status(record) if record is not None else None + + async def cancel_download(self, download_id: str, username: Optional[str] = None, + remove: bool = False) -> bool: + """Mark a download as cancelled. Co-operative — yt-dlp's + progress hook checks shutdown_check on next callback.""" + if self._engine is None: + return False + if self._engine.get_record('soundcloud', download_id) is None: + logger.warning(f"SoundCloud download {download_id} not found") + return False + self._engine.update_record('soundcloud', download_id, {'state': 'Cancelled'}) + logger.info(f"Marked SoundCloud download {download_id} as cancelled") + if remove: + self._engine.remove_record('soundcloud', download_id) + logger.info(f"Removed SoundCloud download {download_id} from queue") + return True + + async def clear_all_completed_downloads(self) -> bool: + if self._engine is None: + return True + terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} + cleared = 0 + for record in list(self._engine.iter_records_for_source('soundcloud')): + if record.get('state') in terminal: + self._engine.remove_record('soundcloud', record['id']) + cleared += 1 + logger.info(f"Cleared {cleared} completed SoundCloud downloads") + return True diff --git a/core/spotify_client.py b/core/spotify_client.py index 50979bd5..09ede9c4 100644 --- a/core/spotify_client.py +++ b/core/spotify_client.py @@ -39,8 +39,8 @@ def _get_min_api_interval(): val = config_manager.get('spotify.min_api_interval', None) if val is not None: return max(0.1, float(val)) # Floor at 100ms to prevent abuse - except Exception: - pass + except Exception as e: + logger.debug("get min_api_interval setting: %s", e) return MIN_API_INTERVAL # Request queuing for burst handling @@ -136,8 +136,20 @@ def _set_global_rate_limit(retry_after_seconds, endpoint_name, has_real_header=F detail=f'{"escalation #" + str(_rate_limit_hit_count) if escalated else "initial"}' f'{", real Retry-After" if has_real_header else ", estimated"}' ) - except Exception: - pass + except Exception as e: + logger.debug("api_call_tracker record rate_limit_ban: %s", e) + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + connected=False, + authenticated=True, + rate_limited=True, + rate_limit=_get_rate_limit_info(), + post_ban_cooldown=_get_post_ban_cooldown_remaining() or None, + ) + except Exception as e: + logger.debug("publish_spotify_status set rate limit: %s", e) def _is_globally_rate_limited(): @@ -210,6 +222,16 @@ def _clear_rate_limit(): _rate_limit_hit_count = 0 _rate_limit_first_hit = 0 logger.info("Global rate limit ban cleared (including post-ban cooldown)") + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + rate_limited=False, + rate_limit=None, + post_ban_cooldown=None, + ) + except Exception as e: + logger.debug("publish_spotify_status clear rate limit: %s", e) def _detect_and_set_rate_limit(exception, endpoint_name="unknown"): @@ -603,13 +625,38 @@ class SpotifyClient: """Check if Spotify client is specifically authenticated (not just iTunes fallback). Results are cached for 60 seconds to avoid excessive API calls. During rate limit bans and post-ban cooldown, returns False without making API calls.""" + rate_limited_state = False if self.sp is None: + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + connected=False, + authenticated=False, + rate_limited=_is_globally_rate_limited(), + rate_limit=_get_rate_limit_info(), + post_ban_cooldown=_get_post_ban_cooldown_remaining() or None, + ) + except Exception as e: + logger.debug("publish_spotify_status no-client: %s", e) return False # If globally rate limited, report as NOT authenticated so callers # skip Spotify and fall through to iTunes fallback naturally. # This prevents any API calls that could extend the ban. if _is_globally_rate_limited(): + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + connected=False, + authenticated=True, + rate_limited=True, + rate_limit=_get_rate_limit_info(), + post_ban_cooldown=_get_post_ban_cooldown_remaining() or None, + ) + except Exception as e: + logger.debug("publish_spotify_status rate-limited: %s", e) return False # Post-ban cooldown: after a ban expires, don't probe Spotify immediately. @@ -618,11 +665,35 @@ class SpotifyClient: if _is_in_post_ban_cooldown(): remaining = _get_post_ban_cooldown_remaining() logger.debug(f"Post-ban cooldown active ({remaining}s left), skipping auth probe") + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + connected=False, + authenticated=True, + rate_limited=False, + rate_limit=None, + post_ban_cooldown=remaining or None, + ) + except Exception as e: + logger.debug("publish_spotify_status post-ban cooldown: %s", e) return False # Check cache first (lock only for brief read) with self._auth_cache_lock: if self._auth_cached_result is not None and (time.time() - self._auth_cache_time) < self._AUTH_CACHE_TTL: + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + connected=self._auth_cached_result, + authenticated=self._auth_cached_result, + rate_limited=False, + rate_limit=None, + post_ban_cooldown=None, + ) + except Exception as e: + logger.debug("publish_spotify_status cache hit: %s", e) return self._auth_cached_result # Cache miss — make API call outside the lock. @@ -637,9 +708,21 @@ class SpotifyClient: with self._auth_cache_lock: self._auth_cached_result = False self._auth_cache_time = time.time() + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + connected=False, + authenticated=False, + rate_limited=False, + rate_limit=None, + post_ban_cooldown=None, + ) + except Exception as e: + logger.debug("publish_spotify_status no-token: %s", e) return False - except Exception: - pass + except Exception as e: + logger.debug("cached token probe: %s", e) # Use a dedicated probe client (retries=0) so a 429 here propagates # immediately and we can detect long Retry-After bans. @@ -668,6 +751,7 @@ class SpotifyClient: ban_duration = max(delay, _BASE_UNKNOWN_BAN) _set_global_rate_limit(ban_duration, 'is_spotify_authenticated', has_real_header=has_real_header) logger.warning(f"Auth probe rate limited — activating {ban_duration}s global ban") + rate_limited_state = True result = True else: logger.debug(f"Spotify authentication check failed: {e}") @@ -677,6 +761,19 @@ class SpotifyClient: self._auth_cached_result = result self._auth_cache_time = time.time() + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + connected=result, + authenticated=result, + rate_limited=rate_limited_state, + rate_limit=_get_rate_limit_info() if rate_limited_state else None, + post_ban_cooldown=None, + ) + except Exception as e: + logger.debug("publish_spotify_status auth probe: %s", e) + return result def disconnect(self): @@ -686,6 +783,18 @@ class SpotifyClient: self.user_id = None self._invalidate_auth_cache() _clear_rate_limit() + try: + from core.metadata.status import publish_spotify_status + + publish_spotify_status( + connected=False, + authenticated=False, + rate_limited=False, + rate_limit=None, + post_ban_cooldown=None, + ) + except Exception as e: + logger.debug("publish_spotify_status disconnect: %s", e) cache_path = 'config/.spotify_cache' try: @@ -1136,8 +1245,8 @@ class SpotifyClient: for raw in cached_results: try: tracks.append(Track.from_spotify_track(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Track.from_spotify_track cache parse: %s", e) if tracks: return tracks @@ -1189,8 +1298,8 @@ class SpotifyClient: for raw in cached_results: try: artists.append(Artist.from_spotify_artist(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Artist.from_spotify_artist cache parse: %s", e) if artists: query_lower = query.lower().strip() artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) @@ -1252,8 +1361,8 @@ class SpotifyClient: for raw in cached_results: try: albums.append(Album.from_spotify_album(raw)) - except Exception: - pass + except Exception as e: + logger.debug("Album.from_spotify_album cache parse: %s", e) if albums: return albums @@ -1527,8 +1636,8 @@ class SpotifyClient: try: albums_list = cached.get('_albums', cached) if isinstance(cached, dict) else cached return [Album.from_spotify_album(ad) for ad in albums_list] - except Exception: - pass # Cache data incompatible, re-fetch + except Exception as e: + logger.debug("artist albums cache reuse: %s", e) if self.is_spotify_authenticated(): try: @@ -1644,6 +1753,31 @@ class SpotifyClient: logger.debug(f"Cannot use fallback for Spotify artist ID: {artist_id}") return None + @rate_limited + def get_artist_top_tracks(self, artist_id: str, country: str = 'US', limit: int = 10) -> List[Dict[str, Any]]: + """Return up to 10 top tracks for an artist (Spotify caps the response at 10). + + Spotify's `artist_top_tracks` endpoint always returns ~10 tracks for the given + market regardless of any limit param. The `limit` argument here is honored as + a UI-side trim only — it never reduces the number of API calls. Tracks come + back in Spotify's standard track-object shape (id, name, artists, album, ...) + so the rest of the pipeline can consume them unchanged. + """ + if not artist_id: + return [] + if not self.is_spotify_authenticated(): + return [] + try: + result = self.sp.artist_top_tracks(artist_id, country=country) + except Exception as e: + _detect_and_set_rate_limit(e, 'get_artist_top_tracks') + logger.warning(f"Spotify artist_top_tracks failed for {artist_id}: {e}") + return [] + if not result: + return [] + tracks = result.get('tracks', []) or [] + return tracks[:max(1, int(limit or 10))] + @rate_limited def get_artists_batch(self, artist_ids: List[str]) -> Dict[str, Dict]: """Get multiple artists, using cache where possible, batch API for misses. diff --git a/core/spotify_worker.py b/core/spotify_worker.py index a3e9cd89..9349bb74 100644 --- a/core/spotify_worker.py +++ b/core/spotify_worker.py @@ -3,12 +3,14 @@ import re import threading import time from difflib import SequenceMatcher +from types import SimpleNamespace from typing import Optional, Dict, Any, List from datetime import datetime, date, timedelta from utils.logging_config import get_logger from database.music_database import MusicDatabase from core.spotify_client import SpotifyClient, SpotifyRateLimitError from core.worker_utils import interruptible_sleep, set_album_api_track_count +from core.enrichment.manual_match_honoring import honor_stored_match logger = get_logger("spotify_worker") @@ -232,8 +234,8 @@ class SpotifyWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null id table resolve failed: %s", e) continue self._process_item(item) @@ -630,11 +632,53 @@ class SpotifyWorker: # ── Individual fallback processing ───────────────────────────────── + def _refresh_album_via_stored_id(self, album_id, stored_id, api_album_dict): + """``honor_stored_match`` callback. Wraps the dict from + ``client.get_album(stored_id)`` in a SimpleNamespace adapter + with the attributes ``_update_album`` reads, then calls it. + Preserves the manual match — never reaches search-by-name.""" + images = api_album_dict.get('images') or [] + image_url = '' + if images and isinstance(images[0], dict): + image_url = images[0].get('url', '') or '' + adapter = SimpleNamespace( + id=api_album_dict.get('id') or stored_id, + name=api_album_dict.get('name', ''), + image_url=image_url, + album_type=api_album_dict.get('album_type', 'album'), + release_date=api_album_dict.get('release_date', ''), + total_tracks=api_album_dict.get('total_tracks', 0), + ) + self._update_album(album_id, adapter) + + def _refresh_track_via_stored_id(self, track_id, stored_id, api_track_dict): + """``honor_stored_match`` callback for tracks. The track-level + update only writes the ID + match status — no metadata + backfill, so the dict shape is irrelevant beyond carrying the + stored ID through.""" + adapter = SimpleNamespace(id=api_track_dict.get('id') or stored_id) + self._update_track_from_search(track_id, adapter) + def _process_album_individual(self, item: Dict[str, Any]): album_id = item['id'] album_name = item['name'] artist_name = item.get('artist', '') + # Issue #501: honor manual matches. If the user has already + # set spotify_album_id on this album row (via match-chip UI), + # refresh metadata via that ID and skip search-by-name — which + # would otherwise overwrite the manual match with whatever + # name-search returned. + if honor_stored_match( + db=self.db, entity_table='albums', entity_id=album_id, + id_column='spotify_album_id', + client_fetch_fn=self.client.get_album, + on_match_fn=self._refresh_album_via_stored_id, + log_prefix='Spotify', + ): + self.stats['matched'] += 1 + return + query = f"{artist_name} {album_name}" if artist_name else album_name results = self.client.search_albums(query, limit=5) @@ -672,6 +716,17 @@ class SpotifyWorker: track_name = item['name'] artist_name = item.get('artist', '') + # Issue #501: honor manual matches (see _process_album_individual). + if honor_stored_match( + db=self.db, entity_table='tracks', entity_id=track_id, + id_column='spotify_track_id', + client_fetch_fn=self.client.get_track_details, + on_match_fn=self._refresh_track_via_stored_id, + log_prefix='Spotify', + ): + self.stats['matched'] += 1 + return + query = f"{artist_name} {track_name}" if artist_name else track_name results = self.client.search_tracks(query, limit=5) diff --git a/core/stats/queries.py b/core/stats/queries.py index df7cfa68..cb505e07 100644 --- a/core/stats/queries.py +++ b/core/stats/queries.py @@ -84,8 +84,8 @@ def get_top_artists(database, image_url_fixer: ImageUrlFixer, time_range: str, l artist['soul_id'] = row[4] finally: conn.close() - except Exception: - pass + except Exception as e: + logger.debug("top artists enrich failed: %s", e) return artists @@ -114,8 +114,8 @@ def get_top_albums(database, image_url_fixer: ImageUrlFixer, time_range: str, li album['artist_id'] = row[2] finally: conn.close() - except Exception: - pass + except Exception as e: + logger.debug("top albums enrich failed: %s", e) return albums @@ -146,8 +146,8 @@ def get_top_tracks(database, image_url_fixer: ImageUrlFixer, time_range: str, li track['artist_id'] = row[2] finally: conn.close() - except Exception: - pass + except Exception as e: + logger.debug("top tracks enrich failed: %s", e) return tracks @@ -172,6 +172,17 @@ def get_db_storage(database) -> dict: return database.get_db_storage_stats() +def get_library_disk_usage(database) -> dict: + """On-disk size of the library, with per-format breakdown. + + Backed by `tracks.file_size` populated during the deep scan from + media-server-reported sizes (Plex MediaPart.size, Jellyfin + MediaSources[].Size, Navidrome , + SoulSync standalone os.path.getsize). + """ + return database.get_library_disk_usage() + + def get_recent_tracks(database, limit: int) -> list[dict]: """Recently played tracks from listening_history.""" conn = database._get_connection() diff --git a/core/streaming/prepare.py b/core/streaming/prepare.py index 44977c66..b22b763d 100644 --- a/core/streaming/prepare.py +++ b/core/streaming/prepare.py @@ -7,9 +7,9 @@ it in the local Stream/ folder for the browser audio player. 1. Reset stream state to 'loading' with the new track info. 2. Clear any prior file from the Stream/ folder (only one stream lives there at a time). -3. Spin up a fresh asyncio event loop and `soulseek_client.download()` +3. Spin up a fresh asyncio event loop and `download_orchestrator.download()` the track. -4. Poll `soulseek_client.get_all_downloads()` every 1.5 s to track +4. Poll `download_orchestrator.get_all_downloads()` every 1.5 s to track progress, with separate handling for queued vs actively downloading states. Queue timeout = 15 s; overall timeout = 60 s. 5. On completion (state ~ 'succeeded' or progress >= 100% AND bytes @@ -45,7 +45,7 @@ logger = logging.getLogger(__name__) class PrepareStreamDeps: """Bundle of cross-cutting deps the stream-prep worker needs.""" config_manager: Any - soulseek_client: Any + download_orchestrator: Any stream_lock: Any # threading.Lock project_root: str # absolute path to web_server.py's directory docker_resolve_path: Callable[[str], str] @@ -112,7 +112,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps): asyncio.set_event_loop(loop) try: - download_result = loop.run_until_complete(deps.soulseek_client.download( + download_result = loop.run_until_complete(deps.download_orchestrator.download( track_data.get('username'), track_data.get('filename'), track_data.get('size', 0) @@ -144,7 +144,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps): try: # Use orchestrator's get_all_downloads() which works for both sources - all_downloads = loop.run_until_complete(deps.soulseek_client.get_all_downloads()) + all_downloads = loop.run_until_complete(deps.download_orchestrator.get_all_downloads()) download_status = deps.find_streaming_download_in_all_downloads(all_downloads, track_data) if download_status: @@ -252,7 +252,7 @@ def prepare_stream_task(track_data, deps: PrepareStreamDeps): download_id = download_status.get('id', '') if download_id and track_data.get('username'): success = loop.run_until_complete( - deps.soulseek_client.signal_download_completion( + deps.download_orchestrator.signal_download_completion( download_id, track_data.get('username'), remove=True) ) if success: diff --git a/core/tag_writer.py b/core/tag_writer.py index cbdfdf11..a1589cc4 100644 --- a/core/tag_writer.py +++ b/core/tag_writer.py @@ -131,8 +131,8 @@ def read_file_tags(file_path: str) -> Dict[str, Any]: result['replaygain_track_peak'] = rg.get('track_peak') result['replaygain_album_gain'] = rg.get('album_gain') result['replaygain_album_peak'] = rg.get('album_peak') - except Exception: - pass + except Exception as e: + logger.debug("read replaygain tags failed: %s", e) return result diff --git a/core/tidal_client.py b/core/tidal_client.py index ce6b0ef7..320a91aa 100644 --- a/core/tidal_client.py +++ b/core/tidal_client.py @@ -640,8 +640,8 @@ class TidalClient: image_id = image_rel.get('id', '') if image_id: image_url = f"https://resources.tidal.com/images/{image_id.replace('-', '/')}/640x640.jpg" - except Exception: - pass + except Exception as _e: + logger.debug("tidal v2 playlists image_url extract: %s", _e) new_playlist = Playlist( id=str(playlist_id), @@ -1188,8 +1188,8 @@ class TidalClient: image_id = image_rel.get('id', '') if image_id: playlist.image_url = f"https://resources.tidal.com/images/{image_id.replace('-', '/')}/640x640.jpg" - except Exception: - pass + except Exception as _e: + logger.debug("tidal playlist image_url extract: %s", _e) logger.info(f"Retrieved Tidal playlist '{playlist.name}' with {len(tracks)} tracks") return playlist diff --git a/core/tidal_download_client.py b/core/tidal_download_client.py index ca76805d..45bdd4b9 100644 --- a/core/tidal_download_client.py +++ b/core/tidal_download_client.py @@ -14,7 +14,6 @@ import re import asyncio import uuid import time -import threading import shutil import subprocess from typing import List, Optional, Dict, Any, Tuple @@ -33,7 +32,7 @@ from utils.logging_config import get_logger from config.settings import config_manager # Import Soulseek data structures for drop-in replacement compatibility -from core.soulseek_client import TrackResult, AlbumResult, DownloadStatus +from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus logger = get_logger("tidal_download_client") @@ -93,7 +92,10 @@ HLS_QUALITY_MAP = { HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"') -class TidalDownloadClient: +from core.download_plugins.base import DownloadSourcePlugin + + +class TidalDownloadClient(DownloadSourcePlugin): """ Tidal download client using tidalapi. Provides search, matching, and download capabilities as a drop-in alternative to YouTube/Soulseek. @@ -116,12 +118,21 @@ class TidalDownloadClient: self.session: Optional['tidalapi.Session'] = None self._init_session() - self.active_downloads: Dict[str, Dict[str, Any]] = {} - self._download_lock = threading.Lock() - self._device_auth_future = None self._device_auth_link = None + # Engine reference is populated by set_engine() at registration + # time. Until then dispatch returns None — orchestrator wires + # this immediately so the only None case is tests that bypass + # the orchestrator. + self._engine = None + + def set_engine(self, engine): + """Engine callback — gives the client access to the central + thread worker + state store. Engine calls this during + ``register_plugin`` if the plugin defines it.""" + self._engine = engine + def set_shutdown_check(self, check_callable): self.shutdown_check = check_callable @@ -432,6 +443,14 @@ class TidalDownloadClient: title=title, album=album_name, track_number=track.track_num, + _source_metadata={ + 'source': 'tidal', + 'track_id': track.id, + 'artist_id': track.artist.id if track.artist else None, + 'isrc': track.isrc or None, + 'bpm': track.bpm if track.bpm and track.bpm > 0 else None, + 'copyright': track.copyright or None, + }, ) return track_result @@ -596,85 +615,36 @@ class TidalDownloadClient: ) from exc async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]: + if '||' not in filename: + logger.error(f"Invalid filename format: {filename}") + return None + if self._engine is None: + # Raise rather than return None so the orchestrator's + # download_with_fallback surfaces a real warning + tries + # the next source. Returning None silently dropped the + # download with no user feedback (per JohnBaumb). + raise RuntimeError("Tidal client has no engine reference — cannot dispatch download") + + track_id_str, display_name = filename.split('||', 1) try: - if '||' not in filename: - logger.error(f"Invalid filename format: {filename}") - return None - - track_id_str, display_name = filename.split('||', 1) - try: - track_id = int(track_id_str) - except ValueError: - logger.error(f"Invalid Tidal track ID: {track_id_str}") - return None - - logger.info(f"Starting Tidal download: {display_name}") - - download_id = str(uuid.uuid4()) - - with self._download_lock: - self.active_downloads[download_id] = { - 'id': download_id, - 'filename': filename, - 'username': 'tidal', - 'state': 'Initializing', - 'progress': 0.0, - 'size': 0, - 'transferred': 0, - 'speed': 0, - 'time_remaining': None, - 'track_id': track_id, - 'display_name': display_name, - 'file_path': None, - } - - download_thread = threading.Thread( - target=self._download_thread_worker, - args=(download_id, track_id, display_name, filename), - daemon=True, - ) - download_thread.start() - - logger.info(f"Tidal download {download_id} started in background") - return download_id - - except Exception as e: - logger.error(f"Failed to start Tidal download: {e}") - import traceback - traceback.print_exc() + track_id = int(track_id_str) + except ValueError: + logger.error(f"Invalid Tidal track ID: {track_id_str}") return None - def _download_thread_worker(self, download_id: str, track_id: int, display_name: str, original_filename: str): - try: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'InProgress, Downloading' + logger.info(f"Starting Tidal download: {display_name}") - file_path = self._download_sync(download_id, track_id, display_name) - - if file_path: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Completed, Succeeded' - self.active_downloads[download_id]['progress'] = 100.0 - self.active_downloads[download_id]['file_path'] = file_path - - logger.info(f"Tidal download {download_id} completed: {file_path}") - else: - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' - - logger.error(f"Tidal download {download_id} failed") - - except Exception as e: - logger.error(f"Tidal download thread failed for {download_id}: {e}") - import traceback - traceback.print_exc() - - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' + return self._engine.worker.dispatch( + source_name='tidal', + target_id=track_id, + display_name=display_name, + original_filename=filename, + impl_callable=self._download_sync, + extra_record_fields={ + 'track_id': track_id, + 'display_name': display_name, + }, + ) def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]: if not self.session or not self.session.check_login(): @@ -719,9 +689,8 @@ class TidalDownloadClient: speed_start = time.time() segments_completed = 0 - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['size'] = 0 + if self._engine is not None: + self._engine.update_record('tidal', download_id, {'size': 0}) with intermediate_path.open('wb') as output_file: if init_uri: @@ -820,97 +789,82 @@ class TidalDownloadClient: def _update_download_progress(self, download_id: str, downloaded: int, segments_completed: int, total_segments: int, speed_start: float): - with self._download_lock: - if download_id not in self.active_downloads: - return - info = self.active_downloads[download_id] - info['transferred'] = downloaded + if self._engine is None: + return + record = self._engine.get_record('tidal', download_id) + if record is None: + return - now = time.time() - elapsed_total = now - speed_start - speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0 - info['speed'] = speed + now = time.time() + elapsed_total = now - speed_start + speed = int(downloaded / elapsed_total) if elapsed_total > 0 else 0 - if total_segments > 0: - progress = (segments_completed / total_segments) * 100 - info['progress'] = round(min(progress, 99.9), 1) + progress = record.get('progress', 0.0) + if total_segments > 0: + progress = round(min((segments_completed / total_segments) * 100, 99.9), 1) - time_remaining = None - if speed > 0: - remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded - if remaining_bytes > 0: - time_remaining = int(remaining_bytes / speed) - info['time_remaining'] = time_remaining + time_remaining = None + if speed > 0: + remaining_bytes = downloaded * (total_segments / max(segments_completed, 1)) - downloaded + if remaining_bytes > 0: + time_remaining = int(remaining_bytes / speed) + + self._engine.update_record('tidal', download_id, { + 'transferred': downloaded, + 'speed': speed, + 'progress': progress, + 'time_remaining': time_remaining, + }) + + def _record_to_status(self, record): + return DownloadStatus( + id=record['id'], + filename=record['filename'], + username=record['username'], + state=record['state'], + progress=record['progress'], + size=record.get('size', 0), + transferred=record.get('transferred', 0), + speed=record.get('speed', 0), + time_remaining=record.get('time_remaining'), + file_path=record.get('file_path'), + ) async def get_all_downloads(self) -> List[DownloadStatus]: - download_statuses = [] - - with self._download_lock: - for _download_id, info in self.active_downloads.items(): - status = DownloadStatus( - id=info['id'], - filename=info['filename'], - username=info['username'], - state=info['state'], - progress=info['progress'], - size=info['size'], - transferred=info['transferred'], - speed=info['speed'], - time_remaining=info.get('time_remaining'), - file_path=info.get('file_path'), - ) - download_statuses.append(status) - - return download_statuses + if self._engine is None: + return [] + return [ + self._record_to_status(record) + for record in self._engine.iter_records_for_source('tidal') + ] async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: - with self._download_lock: - if download_id not in self.active_downloads: - return None - - info = self.active_downloads[download_id] - return DownloadStatus( - id=info['id'], - filename=info['filename'], - username=info['username'], - state=info['state'], - progress=info['progress'], - size=info['size'], - transferred=info['transferred'], - speed=info['speed'], - time_remaining=info.get('time_remaining'), - file_path=info.get('file_path'), - ) + if self._engine is None: + return None + record = self._engine.get_record('tidal', download_id) + return self._record_to_status(record) if record is not None else None async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool: - try: - with self._download_lock: - if download_id not in self.active_downloads: - logger.warning(f"Download {download_id} not found") - return False - - self.active_downloads[download_id]['state'] = 'Cancelled' - logger.info(f"Marked Tidal download {download_id} as cancelled") - - if remove: - del self.active_downloads[download_id] - logger.info(f"Removed Tidal download {download_id} from queue") - - return True - except Exception as e: - logger.error(f"Failed to cancel download {download_id}: {e}") + if self._engine is None: return False + if self._engine.get_record('tidal', download_id) is None: + logger.warning(f"Tidal download {download_id} not found") + return False + self._engine.update_record('tidal', download_id, {'state': 'Cancelled'}) + logger.info(f"Marked Tidal download {download_id} as cancelled") + if remove: + self._engine.remove_record('tidal', download_id) + logger.info(f"Removed Tidal download {download_id} from queue") + return True async def clear_all_completed_downloads(self) -> bool: + if self._engine is None: + return True try: - with self._download_lock: - ids_to_remove = [ - did for did, info in self.active_downloads.items() - if info.get('state', '') in ('Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted') - ] - for did in ids_to_remove: - del self.active_downloads[did] - + terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} + for record in list(self._engine.iter_records_for_source('tidal')): + if record.get('state') in terminal: + self._engine.remove_record('tidal', record['id']) return True except Exception as e: logger.error(f"Error clearing downloads: {e}") diff --git a/core/tidal_worker.py b/core/tidal_worker.py index fa578323..4bcb434a 100644 --- a/core/tidal_worker.py +++ b/core/tidal_worker.py @@ -8,6 +8,7 @@ from utils.logging_config import get_logger from database.music_database import MusicDatabase from core.tidal_client import TidalClient from core.worker_utils import interruptible_sleep +from core.enrichment.manual_match_honoring import honor_stored_match logger = get_logger("tidal_worker") @@ -123,8 +124,8 @@ class TidalWorker: authenticated = False try: authenticated = self.client.is_authenticated() - except Exception: - pass + except Exception as e: + logger.debug("tidal auth status check: %s", e) return { 'enabled': True, @@ -175,8 +176,8 @@ class TidalWorker: itype = item.get('type', '') table = 'artists' if 'artist' in itype else ('albums' if 'album' in itype else 'tracks') # Can't mark status without an ID — just skip - except Exception: - pass + except Exception as e: + logger.debug("null-id item type lookup: %s", e) continue @@ -435,12 +436,30 @@ class TidalWorker: self.stats['not_found'] += 1 logger.debug(f"No match for artist '{artist_name}'") + def _refresh_album_via_stored_id(self, album_id, stored_id, full_album_dict): + """Issue #501 callback. Stored ID exists → fetched full Tidal + album → call ``_update_album`` with the dict in both arg slots + (search-result and full-data shapes overlap on the fields we + need).""" + self._update_album(album_id, full_album_dict, full_album_dict) + + def _refresh_track_via_stored_id(self, track_id, stored_id, full_track_dict): + """Issue #501 callback for tracks — same pattern as albums.""" + self._update_track(track_id, full_track_dict, full_track_dict) + def _process_album(self, album_id: int, album_name: str, artist_name: str, item: Dict[str, Any]): """Process an album: search Tidal, verify, fetch full details, store metadata""" - existing_id = self._get_existing_id('album', album_id) - if existing_id: - logger.debug(f"Preserving existing Tidal ID for album '{album_name}': {existing_id}") - self._mark_status('album', album_id, 'matched') + # Issue #501: honor manual matches. Pre-fix this just marked + # status='matched' without refreshing metadata. Now goes + # through the full refresh path via the stored ID. + if honor_stored_match( + db=self.db, entity_table='albums', entity_id=album_id, + id_column='tidal_id', + client_fetch_fn=self.client.get_album, + on_match_fn=self._refresh_album_via_stored_id, + log_prefix='Tidal', + ): + self.stats['matched'] += 1 return result = self.client.search_album(artist_name, album_name) @@ -486,10 +505,15 @@ class TidalWorker: def _process_track(self, track_id: int, track_name: str, artist_name: str, item: Dict[str, Any]): """Process a track: search Tidal, verify, fetch full details, store metadata""" - existing_id = self._get_existing_id('track', track_id) - if existing_id: - logger.debug(f"Preserving existing Tidal ID for track '{track_name}': {existing_id}") - self._mark_status('track', track_id, 'matched') + # Issue #501: honor manual matches. + if honor_stored_match( + db=self.db, entity_table='tracks', entity_id=track_id, + id_column='tidal_id', + client_fetch_fn=self.client.get_track, + on_match_fn=self._refresh_track_via_stored_id, + log_prefix='Tidal', + ): + self.stats['matched'] += 1 return result = self.client.search_track(artist_name, track_name) diff --git a/core/watchlist/auto_scan.py b/core/watchlist/auto_scan.py index 56550f3b..759d43f7 100644 --- a/core/watchlist/auto_scan.py +++ b/core/watchlist/auto_scan.py @@ -440,8 +440,8 @@ def process_watchlist_scan_automatically(automation_id=None, profile_id=None, de 'new_tracks_found': str(total_new_tracks), 'tracks_added': str(total_added_to_wishlist), }) - except Exception: - pass + except Exception as e: + logger.debug("watchlist_scan_completed emit failed: %s", e) except Exception as e: logger.error(f"Error in automatic watchlist scan: {e}") @@ -460,7 +460,7 @@ def process_watchlist_scan_automatically(automation_id=None, profile_id=None, de # Clear one-time rescan cutoff after full scan cycle try: scanner._clear_rescan_cutoff() - except Exception: + except Exception: # noqa: S110 — finally-block cleanup, logger may be torn down pass # Always reset flag diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 0820aef3..c90e95bd 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -2054,12 +2054,74 @@ class WatchlistScanner: title_variations.append(base_title) unique_title_variations = list(dict.fromkeys(title_variations)) - + # Search for each artist with each title variation from config.settings import config_manager active_server = config_manager.get_active_media_server() allow_duplicates = config_manager.get('wishlist.allow_duplicate_tracks', True) + # Provider-neutral external-ID short-circuit: before doing + # title+artist+album fuzzy comparison, ask the library if any + # row carries a matching external ID (Spotify, Deezer, iTunes, + # Tidal, Qobuz, MusicBrainz, AudioDB, Hydrabase, ISRC). When + # the library has stale album metadata for an existing file + # (e.g. file tagged on the wrong album by an old import), the + # fuzzy block declares the track missing and re-downloads it + # on every scan — but the file's external IDs unambiguously + # identify it as the same recording. See plan-watchlist-id- + # match.md for the reported scenario. + try: + from core.library.track_identity import ( + extract_external_ids, + find_library_track_by_external_id, + find_provenance_by_external_id, + ) + import os as _os_local + # Pass the configured primary source as a hint so the + # extractor can disambiguate raw Spotify / iTunes API + # responses that don't carry a provider / source field + # of their own (Deezer / Discogs / Hydrabase clients + # already tag tracks with _source). + try: + _source_hint = get_primary_source() + except Exception: + _source_hint = None + source_ids = extract_external_ids(track, source_hint=_source_hint) + if source_ids: + matched = find_library_track_by_external_id( + self.database, + external_ids=source_ids, + server_source=active_server, + ) + if matched is not None: + logger.info( + f"[ExtID Match] Track found in library by external ID: " + f"'{original_title}' by '{artists_to_search[0] if artists_to_search else 'Unknown'}' " + f"(matched on: {', '.join(sorted(source_ids.keys()))})" + ) + return False # Track exists in library + + # Second-tier fallback: provenance table. Catches the + # window between "SoulSync downloaded the file" and + # "media-server scan + sync populated the tracks row + # with IDs". File still has to exist on disk — + # otherwise a user who deleted a file would never get + # it back. + prov = find_provenance_by_external_id( + self.database, external_ids=source_ids, + ) + if prov is not None: + prov_path = prov.get('file_path') + if prov_path and _os_local.path.exists(prov_path): + logger.info( + f"[Provenance Match] Track found in download provenance: " + f"'{original_title}' by '{artists_to_search[0] if artists_to_search else 'Unknown'}' " + f"(matched on: {', '.join(sorted(source_ids.keys()))})" + ) + return False + except Exception as ext_id_err: + logger.debug(f"External-ID match probe failed (falling through to fuzzy): {ext_id_err}") + for artist_name in artists_to_search: for query_title in unique_title_variations: # When allow_duplicates is on, skip album hint so we get title+artist matches only @@ -2683,8 +2745,8 @@ class WatchlistScanner: if release_date_str and len(release_date_str) >= 10: release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") is_new = (datetime.now() - release_date).days <= 30 - except Exception: - pass + except Exception as e: + logger.debug("album release_date parse failed: %s", e) for track in tracks: try: @@ -2716,8 +2778,8 @@ class WatchlistScanner: synth_pop += 15 elif age_days <= 365: synth_pop += 5 - except Exception: - pass + except Exception as e: + logger.debug("synthetic popularity age calc failed: %s", e) if similar_artist.occurrence_count >= 3: synth_pop += 10 elif similar_artist.occurrence_count >= 2: @@ -2840,8 +2902,8 @@ class WatchlistScanner: if release_date_str and len(release_date_str) >= 10: release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") is_new = (datetime.now() - release_date).days <= 30 - except Exception: - pass + except Exception as e: + logger.debug("album release_date parse failed: %s", e) for track in tracks: try: @@ -3045,8 +3107,8 @@ class WatchlistScanner: release_date = datetime.strptime(release_date_str, "%Y-%m-%d") days_old = (datetime.now() - release_date).days is_new = days_old <= 30 - except: - pass + except Exception as e: + logger.debug("new-release date parse: %s", e) # Add each track to discovery pool for track in tracks: @@ -3152,8 +3214,8 @@ class WatchlistScanner: elif profile['avg_daily_plays'] > 20: days_lookback = 21 # Heavy listener — keep it fresh logger.info(f"Recent albums window: {days_lookback} days (avg {profile['avg_daily_plays']:.1f} plays/day)") - except Exception: - pass + except Exception as e: + logger.debug("listening profile lookback adjust failed: %s", e) cutoff_date = datetime.now() - timedelta(days=days_lookback) discovery_sources = self._discovery_source_priority() if not discovery_sources: @@ -3436,8 +3498,8 @@ class WatchlistScanner: _artist_genre_cache[_row[0].lower()] = {g.strip().lower() for g in _row[1].split(',') if g.strip()} _conn.close() logger.debug(f"Built genre cache for {len(_artist_genre_cache)} artists") - except Exception: - pass + except Exception as e: + logger.debug("artist genre cache build failed: %s", e) logger.info(f"Curating playlists for sources: {sources_to_process}") @@ -3490,8 +3552,8 @@ class WatchlistScanner: if release_date_str and len(release_date_str) >= 10: release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") days_old = (datetime.now() - release_date).days - except: - pass + except Exception as e: + logger.debug("release-date parse: %s", e) for track in album_data['tracks'].get('items', []): track_id = track.get('id') @@ -3698,8 +3760,8 @@ class WatchlistScanner: _wa_list = self.database.get_watchlist_artists(profile_id=profile_id) for _wa in _wa_list: _wa_id_to_name[str(_wa.id)] = (_wa.artist_name or '').lower() - except Exception: - pass + except Exception as e: + logger.debug("watchlist artist id-to-name map failed: %s", e) all_similar = self.database.get_top_similar_artists(limit=200, profile_id=profile_id) diff --git a/core/web_scan_manager.py b/core/web_scan_manager.py index 810490ce..813fc834 100644 --- a/core/web_scan_manager.py +++ b/core/web_scan_manager.py @@ -19,16 +19,21 @@ class WebScanManager: - Progress tracking and status reporting """ - def __init__(self, media_clients, delay_seconds: int = 60): + def __init__(self, media_server_engine, delay_seconds: int = 60): """ Initialize the web scan manager. Args: - media_clients: Dict containing plex_client, jellyfin_client, navidrome_client + media_server_engine: MediaServerEngine that owns the per-server + clients. Replaces the legacy ``media_clients`` dict — the + manager now resolves the active server's client through + ``self._engine.client(name)`` instead of a hand-keyed + dict that drifted out of sync with the engine's source + of truth. delay_seconds: Debounce delay in seconds (default 60s) """ self.delay = delay_seconds - self.media_clients = media_clients + self._engine = media_server_engine self._timer = None self._scan_in_progress = False self._downloads_during_scan = False @@ -44,29 +49,19 @@ class WebScanManager: logger.info(f"WebScanManager initialized with {delay_seconds}s debounce delay") def _get_active_media_client(self): - """Get the active media client based on config settings""" + """Get the active media client through the engine.""" try: from config.settings import config_manager active_server = config_manager.get_active_media_server() - server_client_map = { - 'jellyfin': 'jellyfin_client', - 'navidrome': 'navidrome_client', - 'plex': 'plex_client', - 'soulsync': 'soulsync_library_client', - } + if not self._engine: + logger.error("Web scan manager has no engine reference") + return None, None - # Try to get the configured active server - if active_server in server_client_map: - client_key = server_client_map[active_server] - client = self.media_clients.get(client_key) - if client and hasattr(client, 'is_connected') and client.is_connected(): - return client, active_server - else: - logger.warning(f"{active_server.title()} client not connected — scan skipped") - return None, None - - logger.error("No active media server configured for scanning") + client = self._engine.client(active_server) + if client and hasattr(client, 'is_connected') and client.is_connected(): + return client, active_server + logger.warning(f"{(active_server or 'unknown').title()} client not connected — scan skipped") return None, None except Exception as e: diff --git a/core/wishlist/presence.py b/core/wishlist/presence.py index 16eddc2b..f7b9cee1 100644 --- a/core/wishlist/presence.py +++ b/core/wishlist/presence.py @@ -3,6 +3,9 @@ from __future__ import annotations import json +import logging + +logger = logging.getLogger(__name__) def load_wishlist_keys(cursor, profile_id: int) -> set[str]: @@ -27,21 +30,21 @@ def load_wishlist_keys(cursor, profile_id: int) -> set[str]: wa = "" if wname: keys.add(wname + "|||" + wa.lower().strip()) - except Exception: - pass + except Exception as e: + logger.debug("parse wishlist row failed: %s", e) try: cursor.execute("SELECT spotify_data FROM wishlist_tracks WHERE profile_id = ?", (profile_id,)) _absorb(cursor.fetchall()) return keys - except Exception: - pass + except Exception as e: + logger.debug("profile-aware wishlist query failed: %s", e) try: cursor.execute("SELECT spotify_data FROM wishlist_tracks") _absorb(cursor.fetchall()) - except Exception: - pass + except Exception as e: + logger.debug("legacy wishlist query failed: %s", e) return keys diff --git a/core/wishlist/processing.py b/core/wishlist/processing.py index e774da4d..542827b3 100644 --- a/core/wishlist/processing.py +++ b/core/wishlist/processing.py @@ -215,8 +215,8 @@ def finalize_auto_wishlist_completion( 'tracks_found': str(tracks_added), 'tracks_failed': str(total_failed - tracks_added), }) - except Exception: - pass + except Exception as e: + logger.debug("emit wishlist_processing_completed failed: %s", e) return completion_summary diff --git a/core/workers/metadata_update.py b/core/workers/metadata_update.py index 085015f1..7b237f0b 100644 --- a/core/workers/metadata_update.py +++ b/core/workers/metadata_update.py @@ -209,8 +209,8 @@ class WebMetadataUpdateWorker: raw = self._db.api_get_artist(best.id) if raw: spotify_artist_id = raw.get('spotify_artist_id') - except Exception: - pass + except Exception as e: + logger.debug("get spotify_artist_id failed: %s", e) return best, has_genres, spotify_artist_id except Exception: return None, False, None @@ -419,10 +419,10 @@ class WebMetadataUpdateWorker: try: for album in artist.albums(): if hasattr(album, 'genres') and album.genres: - album_genres.update(genre.tag if hasattr(genre, 'tag') else str(genre) + album_genres.update(genre.tag if hasattr(genre, 'tag') else str(genre) for genre in album.genres) - except Exception: - pass # Albums might not be accessible + except Exception as e: + logger.debug("artist album genre walk: %s", e) # Combine all genres (prioritize Spotify genres) all_genres = spotify_genres.union(album_genres) diff --git a/core/youtube_client.py b/core/youtube_client.py index 55e042ac..4f1c4b76 100644 --- a/core/youtube_client.py +++ b/core/youtube_client.py @@ -17,7 +17,6 @@ import time import platform import asyncio import uuid -import threading from typing import List, Optional, Dict, Any, Tuple from dataclasses import dataclass from pathlib import Path @@ -34,7 +33,7 @@ from core.matching_engine import MusicMatchingEngine from core.spotify_client import Track as SpotifyTrack # Import Soulseek data structures for drop-in replacement compatibility -from core.soulseek_client import SearchResult, TrackResult, AlbumResult, DownloadStatus +from core.download_plugins.types import SearchResult, TrackResult, AlbumResult, DownloadStatus logger = get_logger("youtube_client") @@ -92,7 +91,10 @@ class YouTubeSearchResult: self.parsed_artist = self.channel -class YouTubeClient: +from core.download_plugins.base import DownloadSourcePlugin + + +class YouTubeClient(DownloadSourcePlugin): """ YouTube download client using yt-dlp. Provides search, matching, and download capabilities with full Spotify metadata integration. @@ -112,10 +114,39 @@ class YouTubeClient: # Callback for shutdown check (avoids circular imports) self.shutdown_check = None - # Rate limiting — serialize YouTube downloads with delay - self._download_semaphore = threading.Semaphore(1) + # Rate-limit policy — applied to engine.worker once the engine + # is wired in via set_engine(). Kept as an attribute for + # backward-compat external readers + so settings reload can + # update it without touching the engine. self._download_delay = config_manager.get('youtube.download_delay', 3) - self._last_download_time = 0 + + # Engine reference is populated by set_engine() at registration + # time. Until then the client can't dispatch downloads — but + # in production the orchestrator wires the engine immediately + # after constructing the registry, so this is only None in + # tests that bypass the orchestrator. + self._engine = None + + def rate_limit_policy(self): + """YouTube reads its download delay from user-tunable config + (``youtube.download_delay``, default 3s). Engine reads this + at ``register_plugin`` time, then ``set_engine`` runs and + re-applies if the config changed since instance construction.""" + from core.download_engine import RateLimitPolicy + return RateLimitPolicy( + download_concurrency=1, + download_delay_seconds=float(self._download_delay), + ) + + def set_engine(self, engine): + """Engine callback — gives the client access to the central + thread worker + state store. Engine calls this during + ``register_plugin`` if the plugin defines it. Worker delay + was already set from rate_limit_policy() — re-apply here so + runtime ``reload_settings`` updates take effect via the + same pathway.""" + self._engine = engine + engine.worker.set_delay('youtube', float(self._download_delay)) def set_shutdown_check(self, check_callable): """Set a callback function to check for system shutdown""" @@ -130,11 +161,6 @@ class YouTubeClient: logger.error("ffmpeg is required but not found") logger.error("The client will attempt to auto-download ffmpeg on first use") - # Download queue management (mirrors Soulseek's download tracking) - # Maps download_id -> download_info dict - self.active_downloads: Dict[str, Dict[str, Any]] = {} - self._download_lock = threading.Lock() # Use threading.Lock for thread safety - # Configure yt-dlp options with bot detection bypass self.download_opts = { 'format': 'bestaudio/best', @@ -272,15 +298,14 @@ class YouTubeClient: self.progress_callback = callback def _progress_hook(self, d): - """ - yt-dlp progress hook - called during download to report progress. - Updates the active_downloads dictionary for the current download. - Mirrors Soulseek's transfer status updates. - """ + """yt-dlp progress hook — called during download to report + progress. Writes to the engine record (Phase C2 lifted state + out of the per-client dict; this hook follows suit).""" try: - # Only update if we have a current download ID if not self.current_download_id: return + if self._engine is None: + return status = d.get('status', 'unknown') @@ -289,24 +314,18 @@ class YouTubeClient: total = d.get('total_bytes') or d.get('total_bytes_estimate', 0) speed = d.get('speed', 0) or 0 eta = d.get('eta', 0) or 0 + percent = (downloaded / total) * 100 if total > 0 else 0 - if total > 0: - percent = (downloaded / total) * 100 - else: - percent = 0 + self._engine.update_record('youtube', self.current_download_id, { + 'state': 'InProgress, Downloading', + 'progress': round(percent, 1), + 'transferred': downloaded, + 'size': total, + 'speed': int(speed), + 'time_remaining': int(eta) if eta > 0 else None, + }) - # Update active downloads dictionary (thread-safe update with lock) - with self._download_lock: - if self.current_download_id in self.active_downloads: - download_info = self.active_downloads[self.current_download_id] - download_info['state'] = 'InProgress, Downloading' # Match Soulseek state format - download_info['progress'] = round(percent, 1) - download_info['transferred'] = downloaded - download_info['size'] = total - download_info['speed'] = int(speed) - download_info['time_remaining'] = int(eta) if eta > 0 else None - - # Also update current_download_progress for legacy compatibility + # Legacy progress dict for any external listeners. self.current_download_progress = { 'status': 'downloading', 'percent': round(percent, 1), @@ -316,30 +335,27 @@ class YouTubeClient: 'eta': int(eta), 'filename': d.get('filename', '') } - - # Call progress callback if set (for UI updates) if self.progress_callback: self.progress_callback(self.current_download_progress) elif status == 'finished': - # Download finished, ffmpeg is converting to MP3 - # Keep state as 'InProgress, Downloading' - the download thread will set final state - with self._download_lock: - if self.current_download_id in self.active_downloads: - self.active_downloads[self.current_download_id]['progress'] = 95.0 # Almost done (converting) - + # Download finished — ffmpeg now converts to MP3. The + # engine.worker thread flips to 'Completed, Succeeded' + # once _download_sync returns; this just bumps progress + # to 95% so the UI doesn't sit at 99.9% during the + # ffmpeg post-process. + self._engine.update_record('youtube', self.current_download_id, { + 'progress': 95.0, + }) self.current_download_progress['status'] = 'postprocessing' self.current_download_progress['percent'] = 95.0 - if self.progress_callback: self.progress_callback(self.current_download_progress) elif status == 'error': - # Mark as error (thread-safe) - with self._download_lock: - if self.current_download_id in self.active_downloads: - self.active_downloads[self.current_download_id]['state'] = 'Errored' - + self._engine.update_record('youtube', self.current_download_id, { + 'state': 'Errored', + }) self.current_download_progress['status'] = 'error' if self.progress_callback: self.progress_callback(self.current_download_progress) @@ -859,137 +875,57 @@ class YouTubeClient: return matches async def download(self, username: str, filename: str, file_size: int = 0) -> Optional[str]: - """ - Download YouTube video as audio (async, Soulseek-compatible interface). + """Download YouTube video as audio. - Returns download_id immediately and runs download in background thread. - Monitor via get_download_status() or get_all_downloads(). + Returns download_id immediately; the actual download runs in + a background thread spawned by ``engine.worker``. Monitor + via ``orchestrator.get_download_status(download_id)``. Args: username: Ignored for YouTube (always "youtube") filename: Encoded as "video_id||title" from search results file_size: Ignored for YouTube (kept for interface compatibility) - - Returns: - download_id: Unique ID for tracking this download """ - try: - # Parse filename to extract video_id - if '||' not in filename: - logger.error(f"Invalid filename format: {filename}") - return None - - video_id, title = filename.split('||', 1) - youtube_url = f"https://www.youtube.com/watch?v={video_id}" - - logger.info(f"Starting YouTube download: {title}") - logger.info(f" URL: {youtube_url}") - - # Create unique download ID - download_id = str(uuid.uuid4()) - - # Initialize download info in active downloads - with self._download_lock: - self.active_downloads[download_id] = { - 'id': download_id, - 'filename': filename, # Keep original encoded format for context matching! - 'username': 'youtube', - 'state': 'Initializing', # Soulseek-style states - 'progress': 0.0, - 'size': file_size or 0, - 'transferred': 0, - 'speed': 0, - 'time_remaining': None, - 'video_id': video_id, - 'url': youtube_url, - 'title': title, - 'file_path': None, # Will be set when download completes - } - - # Start download in background thread (returns immediately) - download_thread = threading.Thread( - target=self._download_thread_worker, - args=(download_id, youtube_url, title, filename), - daemon=True - ) - download_thread.start() - - logger.info(f"YouTube download {download_id} started in background") - return download_id - - except Exception as e: - logger.error(f"Failed to start YouTube download: {e}") - import traceback - traceback.print_exc() + if '||' not in filename: + logger.error(f"Invalid filename format: {filename}") return None + if self._engine is None: + # Raise rather than return None so the orchestrator's + # download_with_fallback surfaces a real warning + tries + # the next source. Returning None silently dropped the + # download with no user feedback (per JohnBaumb). + raise RuntimeError("YouTube client has no engine reference — cannot dispatch download") - def _download_thread_worker(self, download_id: str, youtube_url: str, title: str, original_filename: str): - """ - Background thread worker for downloading YouTube videos. - Updates active_downloads dict with progress. - Serialized via semaphore with configurable delay between downloads. - """ - try: - with self._download_semaphore: - # Enforce delay since last download completed - elapsed = time.time() - self._last_download_time - if self._last_download_time > 0 and elapsed < self._download_delay: - wait_time = self._download_delay - elapsed - logger.info(f"Rate limiting: waiting {wait_time:.1f}s before next YouTube download") - time.sleep(wait_time) + video_id, title = filename.split('||', 1) + youtube_url = f"https://www.youtube.com/watch?v={video_id}" + logger.info("Starting YouTube download: %s (%s)", title, youtube_url) - # Update state to downloading - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'InProgress, Downloading' # Match Soulseek state - - # Set current download ID for progress hook - self.current_download_id = download_id - - # Perform actual download - file_path = self._download_sync(youtube_url, title) - - # Clear current download ID + def _impl(download_id, _target_id, display_name): + # The progress hook reads ``current_download_id`` to know + # which download to update. Set it before the call, clear + # after, even on exception. + self.current_download_id = download_id + try: + return self._download_sync(youtube_url, title) + finally: self.current_download_id = None - # Record completion time for rate limiting - self._last_download_time = time.time() - - if file_path: - # Mark as completed/succeeded (match Soulseek state) - with self._download_lock: - if download_id in self.active_downloads: - # IMPORTANT: Keep original filename for context lookup! - # The filename must match what was used to create the context entry - # We store the actual file path separately - self.active_downloads[download_id]['state'] = 'Completed, Succeeded' # Match Soulseek - self.active_downloads[download_id]['progress'] = 100.0 - self.active_downloads[download_id]['file_path'] = file_path - # DO NOT update filename - keep original_filename for context matching - - logger.info(f"YouTube download {download_id} completed: {file_path}") - else: - # Mark as errored - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' - - logger.error(f"YouTube download {download_id} failed") - - except Exception as e: - logger.error(f"YouTube download thread failed for {download_id}: {e}") - import traceback - traceback.print_exc() - - # Mark as errored - with self._download_lock: - if download_id in self.active_downloads: - self.active_downloads[download_id]['state'] = 'Errored' - - # Clear current download ID - if self.current_download_id == download_id: - self.current_download_id = None + return self._engine.worker.dispatch( + source_name='youtube', + target_id=video_id, + display_name=title, + original_filename=filename, + impl_callable=_impl, + extra_record_fields={ + 'video_id': video_id, + 'url': youtube_url, + 'title': title, + }, + ) + # Legacy worker stub kept temporarily for legacy comment context — + # see _download_sync below for the actual yt-dlp invocation that + # the engine's BackgroundDownloadWorker now drives. def _download_sync(self, youtube_url: str, title: str) -> Optional[str]: """ Synchronous download method (runs in thread pool executor). @@ -1138,123 +1074,76 @@ class YouTubeClient: traceback.print_exc() return None + def _record_to_status(self, record): + """Translate an engine record dict into the DownloadStatus + dataclass shape consumers expect.""" + return DownloadStatus( + id=record['id'], + filename=record['filename'], + username=record['username'], + state=record['state'], + progress=record['progress'], + size=record.get('size', 0), + transferred=record.get('transferred', 0), + speed=record.get('speed', 0), + time_remaining=record.get('time_remaining'), + file_path=record.get('file_path'), + ) + async def get_all_downloads(self) -> List[DownloadStatus]: - """ - Get all active downloads (matches Soulseek interface). - - Returns: - List of DownloadStatus objects for all active downloads - """ - download_statuses = [] - - with self._download_lock: - for _download_id, download_info in self.active_downloads.items(): - status = DownloadStatus( - id=download_info['id'], - filename=download_info['filename'], - username=download_info['username'], - state=download_info['state'], - progress=download_info['progress'], - size=download_info['size'], - transferred=download_info['transferred'], - speed=download_info['speed'], - time_remaining=download_info.get('time_remaining') - ) - download_statuses.append(status) - - return download_statuses + """Active downloads owned by the YouTube source — read from + engine state.""" + if self._engine is None: + return [] + return [ + self._record_to_status(record) + for record in self._engine.iter_records_for_source('youtube') + ] async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: - """ - Get status of a specific download (matches Soulseek interface). - - Args: - download_id: Download ID to query - - Returns: - DownloadStatus object or None if not found - """ - with self._download_lock: - if download_id not in self.active_downloads: - return None - - download_info = self.active_downloads[download_id] - - return DownloadStatus( - id=download_info['id'], - filename=download_info['filename'], - username=download_info['username'], - state=download_info['state'], - progress=download_info['progress'], - size=download_info['size'], - transferred=download_info['transferred'], - speed=download_info['speed'], - time_remaining=download_info.get('time_remaining'), - file_path=download_info.get('file_path') - ) - + """Single download status — read from engine state. Returns + None if this id isn't owned by YouTube (or not found).""" + if self._engine is None: + return None + record = self._engine.get_record('youtube', download_id) + if record is None: + return None + return self._record_to_status(record) async def clear_all_completed_downloads(self) -> bool: - """ - Clear all terminal (completed, cancelled, errored) downloads from the list. - Matches Soulseek interface. - """ + """Clear terminal-state downloads (Completed / Cancelled / + Errored / Aborted) from engine state.""" + if self._engine is None: + return True try: - with self._download_lock: - # Identify IDs to remove - ids_to_remove = [] - for download_id, info in self.active_downloads.items(): - state = info.get('state', '') - # Check for terminal states - # Note: We check exact strings used in _download_thread_worker and cancel_download - if state in ['Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted']: - ids_to_remove.append(download_id) - - # Remove them - for download_id in ids_to_remove: - del self.active_downloads[download_id] - logger.debug(f"Cleared finished download {download_id}") - + terminal_states = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} + for record in list(self._engine.iter_records_for_source('youtube')): + if record.get('state') in terminal_states: + self._engine.remove_record('youtube', record['id']) + logger.debug("Cleared finished YouTube download %s", record['id']) return True except Exception as e: logger.error(f"Error clearing downloads: {e}") return False async def cancel_download(self, download_id: str, username: str = None, remove: bool = False) -> bool: - """ - Cancel an active download (matches Soulseek interface). - - NOTE: YouTube downloads cannot be truly cancelled mid-download, - but we mark them as cancelled for UI consistency. - - Args: - download_id: Download ID to cancel - username: Ignored for YouTube (kept for interface compatibility) - remove: If True, remove from active downloads after cancelling - - Returns: - True if cancelled successfully, False otherwise - """ - try: - with self._download_lock: - if download_id not in self.active_downloads: - logger.warning(f"Download {download_id} not found") - return False - - # Update state to cancelled - self.active_downloads[download_id]['state'] = 'Cancelled' - logger.info(f"Marked YouTube download {download_id} as cancelled") - - # Remove from active downloads if requested - if remove: - del self.active_downloads[download_id] - logger.info(f"Removed YouTube download {download_id} from queue") - - return True - - except Exception as e: - logger.error(f"Failed to cancel download {download_id}: {e}") + """Mark a YouTube download as cancelled. yt-dlp downloads + can't be truly interrupted mid-stream — this only flips + the state for UI consistency. ``remove=True`` also drops + the engine record.""" + if self._engine is None: return False + record = self._engine.get_record('youtube', download_id) + if record is None: + logger.warning(f"YouTube download {download_id} not found") + return False + + self._engine.update_record('youtube', download_id, {'state': 'Cancelled'}) + logger.info(f"Marked YouTube download {download_id} as cancelled") + if remove: + self._engine.remove_record('youtube', download_id) + logger.info(f"Removed YouTube download {download_id} from queue") + return True def _enhance_metadata(self, filepath: str, spotify_track: Optional[SpotifyTrack], yt_result: YouTubeSearchResult, track_number: int = 1, disc_number: int = 1, release_year: str = None, artist_genres: list = None): """ @@ -1301,8 +1190,8 @@ class YouTubeClient: album_data = track_details.get('album', {}) if album_data.get('artists'): album_artist = album_data['artists'][0] - except: - pass + except Exception as e: + logger.debug("spotify album artist lookup: %s", e) logger.debug(" Setting metadata tags...") diff --git a/database/music_database.py b/database/music_database.py index 2fbb8cde..857d16b0 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -294,6 +294,7 @@ class MusicDatabase: duration INTEGER, -- milliseconds file_path TEXT, bitrate INTEGER, + file_size INTEGER, -- bytes; populated by deep scan from media-server API created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (album_id) REFERENCES albums (id) ON DELETE CASCADE, @@ -649,8 +650,8 @@ class MusicDatabase: try: cursor.execute("ALTER TABLE sync_history ADD COLUMN track_results TEXT") logger.info("Added track_results column to sync_history table") - except Exception: - pass + except Exception as e: + logger.debug("Failed to add track_results column: %s", e) # Migration: add source_page column to sync_history (UI origin context for batch panel) try: @@ -659,8 +660,8 @@ class MusicDatabase: try: cursor.execute("ALTER TABLE sync_history ADD COLUMN source_page TEXT") logger.info("Added source_page column to sync_history table") - except Exception: - pass + except Exception as e: + logger.debug("Failed to add source_page column: %s", e) # Migration: add track_artist column for per-track artist on compilations/DJ mixes try: @@ -669,8 +670,25 @@ class MusicDatabase: try: cursor.execute("ALTER TABLE tracks ADD COLUMN track_artist TEXT") logger.info("Added track_artist column to tracks table") - except Exception: - pass + except Exception as e: + logger.debug("Failed to add track_artist column: %s", e) + + # Migration: add file_size column so the Stats page can show + # total library size on disk without having to walk the + # filesystem on every request. Populated by the deep scan from + # whatever the media server reports (Plex MediaPart.size, + # Jellyfin MediaSources[].Size, Navidrome , + # SoulSync standalone os.path.getsize). NULL on existing rows + # until the next deep scan fills them in — UI handles the + # NULL case by showing "(run a Deep Scan to populate)". + try: + cursor.execute("SELECT file_size FROM tracks LIMIT 1") + except Exception: + try: + cursor.execute("ALTER TABLE tracks ADD COLUMN file_size INTEGER") + logger.info("Added file_size column to tracks table") + except Exception as e: + logger.debug("Failed to add file_size column: %s", e) # One-time migration: purge discovery cache entries that lack track_number. # Prior versions cached discovery results without track_number/disc_number/release_date, @@ -686,8 +704,8 @@ class MusicDatabase: cursor.execute("CREATE TABLE _discovery_cache_v2_migrated (applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") if purged > 0: logger.info(f"Purged {purged} stale discovery cache entries (missing track_number)") - except Exception: - pass + except Exception as e: + logger.debug("Failed to purge stale discovery cache entries: %s", e) # One-time migration: purge Deezer album/track cache entries with missing data. # Deezer's /artist/{id}/albums returns albums without artist info, and search @@ -703,8 +721,8 @@ class MusicDatabase: cursor.execute("CREATE TABLE _deezer_cache_v2_migrated (applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") if purged > 0: logger.info(f"Purged {purged} stale Deezer cache entries (missing artist/track_position)") - except Exception: - pass + except Exception as e: + logger.debug("Failed to purge stale Deezer cache entries: %s", e) # One-time migration: purge cached tracks/albums with junk artist names. # The cache gate now rejects these, but existing entries need cleaning. @@ -720,8 +738,8 @@ class MusicDatabase: cursor.execute("CREATE TABLE _cache_junk_artist_purged (applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") if purged > 0: logger.info(f"Purged {purged} cached tracks/albums with junk artist names") - except Exception: - pass + except Exception as e: + logger.debug("Failed to purge cached tracks/albums with junk artist names: %s", e) # HiFi API instances table cursor.execute(""" @@ -801,8 +819,8 @@ class MusicDatabase: config = json.loads(row[2]) if row[2] else {} then_actions = json.dumps([{'type': row[1], 'config': config}]) cursor.execute("UPDATE automations SET then_actions = ? WHERE id = ?", (then_actions, row[0])) - except Exception: - pass + except Exception as e: + logger.debug("Failed to migrate notify data for automation row: %s", e) logger.info("Migrated existing notify data to then_actions") except Exception as e: logger.error(f"Error adding automation then_actions column: {e}") @@ -1384,6 +1402,55 @@ class MusicDatabase: cursor.execute("ALTER TABLE track_downloads ADD COLUMN bitrate INTEGER") logger.info("Added audio detail columns (bit_depth, sample_rate, bitrate) to track_downloads") + # Migration: Add external metadata-source ID columns to + # track_downloads. Persists the IDs we already collect at + # post-processing time so the watchlist scanner + media-server + # sync backfill can read them without waiting for the async + # enrichment workers. + external_id_cols = [ + 'spotify_track_id', 'itunes_track_id', 'deezer_track_id', + 'tidal_track_id', 'qobuz_track_id', 'musicbrainz_recording_id', + 'audiodb_id', 'soul_id', 'isrc', + ] + added_external = False + for _col in external_id_cols: + if _col not in td_columns: + cursor.execute(f"ALTER TABLE track_downloads ADD COLUMN {_col} TEXT") + added_external = True + if added_external: + logger.info(f"Added external-ID columns to track_downloads: {', '.join(external_id_cols)}") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_spotify_id ON track_downloads (spotify_track_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_itunes_id ON track_downloads (itunes_track_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_deezer_id ON track_downloads (deezer_track_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_tidal_id ON track_downloads (tidal_track_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_qobuz_id ON track_downloads (qobuz_track_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_mbid ON track_downloads (musicbrainz_recording_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_audiodb_id ON track_downloads (audiodb_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_soul_id ON track_downloads (soul_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_td_isrc ON track_downloads (isrc)") + + # Persistent MusicBrainz album → release-MBID cache. + # Backs `core/metadata/album_mbid_cache.py`. Keyed by the same + # (normalized_album_key, artist_key) shape the in-memory + # `mb_release_cache` uses, so a successful lookup remembered + # ONCE applies to every future track of the same album for + # the install's lifetime. Solves the "tracks of one album get + # different release MBIDs after cache eviction / restart" + # issue that causes Navidrome to split albums. + cursor.execute(""" + CREATE TABLE IF NOT EXISTS mb_album_release_cache ( + normalized_album_key TEXT NOT NULL, + artist_key TEXT NOT NULL, + release_mbid TEXT NOT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (normalized_album_key, artist_key) + ) + """) + cursor.execute( + "CREATE INDEX IF NOT EXISTS idx_mb_album_release_mbid " + "ON mb_album_release_cache (release_mbid)" + ) + # Discovery artist blacklist — artists users never want to see in discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS discovery_artist_blacklist ( @@ -1436,6 +1503,7 @@ class MusicDatabase: spotify_album_id TEXT, tidal_album_id TEXT, deezer_album_id TEXT, + discogs_release_id TEXT, image_url TEXT, release_date TEXT, total_tracks INTEGER DEFAULT 0, @@ -1450,6 +1518,18 @@ class MusicDatabase: cursor.execute("CREATE INDEX IF NOT EXISTS idx_lalp_profile ON liked_albums_pool (profile_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_lalp_spotify ON liked_albums_pool (spotify_album_id)") + # Migration: add discogs_release_id column for the Discogs + # collection source on the Your Albums section. Idempotent — + # safe on existing installs that already have the table. + try: + cursor.execute("SELECT discogs_release_id FROM liked_albums_pool LIMIT 1") + except Exception: + try: + cursor.execute("ALTER TABLE liked_albums_pool ADD COLUMN discogs_release_id TEXT") + logger.info("Added discogs_release_id column to liked_albums_pool") + except Exception as e: + logger.debug("Failed to add discogs_release_id column: %s", e) + logger.info("Discovery tables added/verified successfully") except Exception as e: @@ -2388,8 +2468,8 @@ class MusicDatabase: if 'profile_id' not in columns: cursor.execute(f"ALTER TABLE {table} ADD COLUMN profile_id INTEGER DEFAULT 1") logger.info(f"Repaired missing profile_id column on {table}") - except Exception: - pass + except Exception as e: + logger.debug("Failed to repair profile_id column on %s: %s", table, e) if already_migrated: return # Rest of migration already done @@ -2586,8 +2666,8 @@ class MusicDatabase: for idx_name, table in index_pairs: try: cursor.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table} (profile_id)") - except Exception: - pass + except Exception as e: + logger.debug("Failed to create index %s on %s: %s", idx_name, table, e) # Set migration marker cursor.execute(""" @@ -3246,8 +3326,8 @@ class MusicDatabase: )) if cursor.rowcount > 0: inserted += 1 - except Exception: - pass + except Exception as e: + logger.debug("Failed to insert listening event: %s", e) conn.commit() return inserted except Exception as e: @@ -3555,8 +3635,8 @@ class MusicDatabase: total_size = 0 try: total_size = os.path.getsize(str(self.database_path)) - except Exception: - pass + except Exception as e: + logger.debug("Failed to stat database file size: %s", e) conn = self._get_connection() cursor = conn.cursor() @@ -3583,8 +3663,8 @@ class MusicDatabase: cursor.execute(f"SELECT COUNT(*) FROM [{tbl}]") count = cursor.fetchone()[0] tables.append({'name': tbl, 'size': count}) - except Exception: - pass + except Exception as e: + logger.debug("Failed to get row count for table %s: %s", tbl, e) tables.sort(key=lambda x: x['size'], reverse=True) return { @@ -3599,6 +3679,85 @@ class MusicDatabase: if conn: conn.close() + def get_library_disk_usage(self): + """Aggregate disk usage of the on-disk music library. + + Returns: + { + 'total_bytes': int, # sum of all known file sizes + 'tracks_with_size': int, # count of tracks with a known size + 'tracks_without_size': int, # count of tracks where size is NULL + 'by_format': { # bytes per file extension + 'flac': int, 'mp3': int, ... + }, + 'has_data': bool, # False on fresh installs / before first deep scan + } + + Returns the empty-shape dict when the column doesn't exist (very + old install pre-migration) — UI shows "(run a Deep Scan)" in + that case rather than crashing. + """ + empty = { + 'total_bytes': 0, + 'tracks_with_size': 0, + 'tracks_without_size': 0, + 'by_format': {}, + 'has_data': False, + } + conn = None + try: + conn = self._get_connection() + cursor = conn.cursor() + + # Confirm column exists (defensive against fresh-install race + # where the migration hasn't run yet). + try: + cursor.execute("SELECT file_size FROM tracks LIMIT 1") + except Exception: + return empty + + cursor.execute( + "SELECT COALESCE(SUM(file_size), 0), " + " COUNT(file_size), " + " COUNT(*) - COUNT(file_size) " + "FROM tracks" + ) + row = cursor.fetchone() + total_bytes = int(row[0] or 0) + tracks_with_size = int(row[1] or 0) + tracks_without_size = int(row[2] or 0) + + # Per-format breakdown via Python aggregation. Doing the + # extension split in SQLite is fragile (paths with dots + # before the file extension would group wrong); doing it + # in Python is one os.path.splitext per row, which is + # negligible cost compared to the SUM() above. + cursor.execute( + "SELECT file_path, file_size FROM tracks " + "WHERE file_size IS NOT NULL AND file_path IS NOT NULL " + " AND file_path != ''" + ) + by_format: dict = {} + for path, size in cursor.fetchall(): + ext = os.path.splitext(path)[1].lstrip('.').lower() + if not ext or len(ext) > 6: + continue + by_format[ext] = by_format.get(ext, 0) + int(size or 0) + + return { + 'total_bytes': total_bytes, + 'tracks_with_size': tracks_with_size, + 'tracks_without_size': tracks_without_size, + 'by_format': by_format, + 'has_data': tracks_with_size > 0, + } + except Exception as e: + logger.error(f"Error getting library disk usage: {e}") + return empty + finally: + if conn: + conn.close() + @staticmethod def _listening_time_filter(time_range, alias=''): """Build a WHERE clause for time-range filtering.""" @@ -4030,8 +4189,8 @@ class MusicDatabase: 'bubble_snapshots', 'recent_releases']: try: cursor.execute(f"DELETE FROM {table} WHERE profile_id = ?", (profile_id,)) - except Exception: - pass + except Exception as e: + logger.debug("Failed to delete from %s for profile: %s", table, e) cursor.execute("DELETE FROM profiles WHERE id = ?", (profile_id,)) conn.commit() return cursor.rowcount > 0 @@ -4921,12 +5080,20 @@ class MusicDatabase: # Get file path and media info (Plex-specific, Jellyfin may not have these) file_path = None bitrate = None + file_size = None if hasattr(track_obj, 'media') and track_obj.media: media = track_obj.media[0] if track_obj.media else None if media: if hasattr(media, 'parts') and media.parts: part = media.parts[0] file_path = getattr(part, 'file', None) + # Plex's MediaPart exposes the file size in bytes + # via plexapi — pull it for the Library Disk + # Usage card on Stats. None when the server + # didn't report a size. + _plex_size = getattr(part, 'size', None) + if isinstance(_plex_size, int) and _plex_size > 0: + file_size = _plex_size bitrate = getattr(media, 'bitrate', None) # Fallback for Navidrome/Subsonic tracks @@ -4936,6 +5103,14 @@ class MusicDatabase: bitrate = track_obj.bitRate if file_path is None and hasattr(track_obj, 'suffix') and track_obj.suffix: file_path = f"{track_obj.title}.{track_obj.suffix}" + # File size: Jellyfin / Navidrome / SoulSync-standalone + # all set track_obj.file_size on their wrapper class. + # Plex came in via the media.parts[0].size path above — + # don't clobber that. + if file_size is None and hasattr(track_obj, 'file_size'): + _wrapper_size = getattr(track_obj, 'file_size', None) + if isinstance(_wrapper_size, int) and _wrapper_size > 0: + file_size = _wrapper_size # Extract per-track artist for compilations/DJ mixes. # Only stored when it differs from the album artist. @@ -4977,8 +5152,8 @@ class MusicDatabase: album_artist_name = artist_row[0] if artist_row else '' if nav_artist and nav_artist.lower() != album_artist_name.lower(): track_artist = nav_artist - except Exception: - pass + except Exception as e: + logger.debug("Failed to load album artist for track_artist comparison: %s", e) # Extract MusicBrainz recording ID from server if available (Navidrome provides this) mbid = getattr(track_obj, 'musicBrainzId', None) or None @@ -4991,24 +5166,43 @@ class MusicDatabase: if is_new_track: cursor.execute(""" INSERT INTO tracks - (id, album_id, artist_id, title, track_number, duration, file_path, bitrate, server_source, track_artist, musicbrainz_recording_id, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) - """, (track_id, album_id, artist_id, title, track_number, duration, file_path, bitrate, server_source, track_artist, mbid)) + (id, album_id, artist_id, title, track_number, duration, file_path, bitrate, file_size, server_source, track_artist, musicbrainz_recording_id, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + """, (track_id, album_id, artist_id, title, track_number, duration, file_path, bitrate, file_size, server_source, track_artist, mbid)) else: # Update server-provided fields only — preserves spotify_track_id, deezer_id, - # isrc, bpm, and all other enrichment data + # isrc, bpm, and all other enrichment data. file_size uses + # COALESCE(?, file_size) so a NULL from the server (e.g. + # Jellyfin sometimes omits Size on first sync) doesn't wipe + # an existing value. cursor.execute(""" UPDATE tracks SET album_id = ?, artist_id = ?, title = ?, track_number = ?, - duration = ?, file_path = ?, bitrate = ?, server_source = ?, + duration = ?, file_path = ?, bitrate = ?, + file_size = COALESCE(?, file_size), + server_source = ?, track_artist = COALESCE(?, track_artist), musicbrainz_recording_id = COALESCE(?, musicbrainz_recording_id), updated_at = CURRENT_TIMESTAMP WHERE id = ? - """, (album_id, artist_id, title, track_number, duration, file_path, bitrate, server_source, track_artist, mbid, track_id)) + """, (album_id, artist_id, title, track_number, duration, file_path, bitrate, file_size, server_source, track_artist, mbid, track_id)) conn.commit() + # Backfill external metadata-source IDs from track_downloads + # provenance. SoulSync collected them at download time but the + # media-server scan can't see them — without this hook, + # tracks.spotify_track_id / itunes_track_id / etc. stay empty + # until the async enrichment workers eventually catch up + # (hours later), during which window the watchlist scanner + # treats freshly downloaded files as missing and re-downloads + # them. Idempotent COALESCE on each column preserves any value + # the enrichment worker already wrote. + try: + self.backfill_track_external_ids_from_provenance(track_id, file_path) + except Exception as backfill_err: + logger.debug(f"Provenance ID backfill skipped for track {track_id}: {backfill_err}") + # Log new imports to library history if is_new_track: try: @@ -5025,8 +5219,8 @@ class MusicDatabase: file_path=file_path, thumb_url=album_row[1] if album_row and len(album_row) > 1 else None ) - except Exception: - pass # Non-critical history logging + except Exception as e: + logger.debug("history logging: %s", e) return True @@ -9786,7 +9980,8 @@ class MusicDatabase: if source_id and source_id_type: col = {'spotify': 'spotify_album_id', 'tidal': 'tidal_album_id', - 'deezer': 'deezer_album_id'}.get(source_id_type) + 'deezer': 'deezer_album_id', + 'discogs': 'discogs_release_id'}.get(source_id_type) if col: set_parts.append(f"{col} = COALESCE({col}, ?)") params.append(source_id) @@ -9808,7 +10003,8 @@ class MusicDatabase: else: sources_json = json.dumps([source_service]) id_cols = {'spotify': 'spotify_album_id', 'tidal': 'tidal_album_id', - 'deezer': 'deezer_album_id'} + 'deezer': 'deezer_album_id', + 'discogs': 'discogs_release_id'} col_values = {v: None for v in id_cols.values()} if source_id and source_id_type and source_id_type in id_cols: col_values[id_cols[source_id_type]] = source_id @@ -9816,13 +10012,13 @@ class MusicDatabase: cursor.execute(""" INSERT INTO liked_albums_pool (album_name, artist_name, normalized_key, spotify_album_id, tidal_album_id, - deezer_album_id, image_url, release_date, total_tracks, source_services, - profile_id, last_fetched_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + deezer_album_id, discogs_release_id, image_url, release_date, total_tracks, + source_services, profile_id, last_fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) """, ( album_name, artist_name, normalized, col_values['spotify_album_id'], col_values['tidal_album_id'], - col_values['deezer_album_id'], + col_values['deezer_album_id'], col_values['discogs_release_id'], image_url, release_date, total_tracks or 0, sources_json, profile_id )) @@ -9914,8 +10110,24 @@ class MusicDatabase: source_filename: str, source_size: int = 0, audio_quality: str = '', track_title: str = '', track_artist: str = '', track_album: str = '', status: str = 'completed', track_id: str = None, - bit_depth: int = None, sample_rate: int = None, bitrate: int = None) -> Optional[int]: - """Record a download with full source provenance. Returns the record ID.""" + bit_depth: int = None, sample_rate: int = None, bitrate: int = None, + spotify_track_id: Optional[str] = None, + itunes_track_id: Optional[str] = None, + deezer_track_id: Optional[str] = None, + tidal_track_id: Optional[str] = None, + qobuz_track_id: Optional[str] = None, + musicbrainz_recording_id: Optional[str] = None, + audiodb_id: Optional[str] = None, + soul_id: Optional[str] = None, + isrc: Optional[str] = None) -> Optional[int]: + """Record a download with full source provenance. Returns the record ID. + + External-ID kwargs (spotify_track_id et al.) capture the metadata- + source identity that the user originally asked for — they're written + at download time so the watchlist scanner can recognize the file as + already present without waiting for the async enrichment workers + to backfill them onto the ``tracks`` row. + """ try: conn = self._get_connection() cursor = conn.cursor() @@ -9941,17 +10153,123 @@ class MusicDatabase: INSERT INTO track_downloads (track_id, file_path, source_service, source_username, source_filename, source_size, audio_quality, track_title, track_artist, track_album, status, - bit_depth, sample_rate, bitrate) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + bit_depth, sample_rate, bitrate, + spotify_track_id, itunes_track_id, deezer_track_id, tidal_track_id, + qobuz_track_id, musicbrainz_recording_id, audiodb_id, soul_id, isrc) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, (track_id, file_path, source_service, source_username, source_filename, source_size, audio_quality, track_title, track_artist, track_album, status, - bit_depth, sample_rate, bitrate)) + bit_depth, sample_rate, bitrate, + spotify_track_id, itunes_track_id, deezer_track_id, tidal_track_id, + qobuz_track_id, musicbrainz_recording_id, audiodb_id, soul_id, isrc)) conn.commit() return cursor.lastrowid except Exception as e: logger.error(f"Error recording track download: {e}") return None + def get_provenance_by_file_path(self, file_path: str) -> Optional[Dict[str, Any]]: + """Return the most recent track_downloads row matching ``file_path``. + + Tries exact match first, then a basename-suffix LIKE fallback for + cases where the media-server scan reports the file at a slightly + different path than what was recorded at download time (Windows + separators, symlink resolution, container mount-root differences). + """ + if not file_path: + return None + try: + conn = self._get_connection() + cursor = conn.cursor() + cursor.execute( + "SELECT * FROM track_downloads WHERE file_path = ? ORDER BY id DESC LIMIT 1", + (file_path,), + ) + row = cursor.fetchone() + if row is None: + import os as _os + fname = _os.path.basename(file_path.replace('\\', '/')) + if fname: + cursor.execute( + "SELECT * FROM track_downloads WHERE file_path LIKE ? OR file_path LIKE ? " + "ORDER BY id DESC LIMIT 1", + (f'%/{fname}', f'%\\{fname}'), + ) + row = cursor.fetchone() + if row is None: + return None + try: + return dict(row) + except (TypeError, ValueError): + cols = [c[0] for c in cursor.description] + return dict(zip(cols, row, strict=False)) + except Exception as exc: + logger.debug(f"get_provenance_by_file_path failed: {exc}") + return None + + def backfill_track_external_ids_from_provenance(self, track_id: str, file_path: Optional[str]) -> int: + """Copy external IDs from ``track_downloads`` onto a ``tracks`` row. + + Idempotent: only writes columns that are currently NULL/empty on + the tracks row AND have a value in the provenance row. Returns the + number of columns updated. Called from + ``insert_or_update_media_track`` immediately after the row is + inserted/updated so freshly synced media-server rows pick up + whatever IDs SoulSync already knew at download time. + """ + if not track_id or not file_path: + return 0 + prov = self.get_provenance_by_file_path(file_path) + if not prov: + return 0 + + # Map provenance column -> tracks column. Different naming + # conventions because tracks.* uses shorter names (``deezer_id``, + # ``tidal_id``, ``qobuz_id``) while track_downloads uses the + # explicit ``_track_id`` suffix to avoid ambiguity. + prov_to_tracks = { + 'spotify_track_id': 'spotify_track_id', + 'itunes_track_id': 'itunes_track_id', + 'deezer_track_id': 'deezer_id', + 'tidal_track_id': 'tidal_id', + 'qobuz_track_id': 'qobuz_id', + 'musicbrainz_recording_id': 'musicbrainz_recording_id', + 'audiodb_id': 'audiodb_id', + 'soul_id': 'soul_id', + 'isrc': 'isrc', + } + + updates: Dict[str, str] = {} + for prov_col, track_col in prov_to_tracks.items(): + val = prov.get(prov_col) + if not val: + continue + updates[track_col] = str(val) + if not updates: + return 0 + + try: + conn = self._get_connection() + cursor = conn.cursor() + # Coalesce-update: only fill empty columns. Preserves any IDs + # the enrichment worker already populated (those are usually + # more reliable than provenance for non-primary sources). + set_clauses = [] + params = [] + for track_col, val in updates.items(): + set_clauses.append(f"{track_col} = COALESCE(NULLIF({track_col}, ''), ?)") + params.append(val) + params.append(track_id) + cursor.execute( + f"UPDATE tracks SET {', '.join(set_clauses)} WHERE id = ?", + params, + ) + conn.commit() + return cursor.rowcount or 0 + except Exception as exc: + logger.debug(f"backfill_track_external_ids_from_provenance failed: {exc}") + return 0 + def get_track_downloads(self, track_id: str) -> list: """Get all download records for a library track.""" try: @@ -10583,8 +10901,8 @@ class MusicDatabase: """) for row in cursor.fetchall(): source_counts[row['download_source']] = row['cnt'] - except Exception: - pass + except Exception as e: + logger.debug("Failed to load library history source counts: %s", e) stats['source_counts'] = source_counts return stats @@ -10875,8 +11193,8 @@ class MusicDatabase: WHERE playlist_id = ? AND source_track_id IS NOT NULL AND extra_data IS NOT NULL """, (playlist_id,)) old_extra_map = {row['source_track_id']: row['extra_data'] for row in cursor.fetchall()} - except Exception: - pass + except Exception as e: + logger.debug("Failed to preserve mirrored playlist extra_data: %s", e) # Replace all tracks cursor.execute("DELETE FROM mirrored_playlist_tracks WHERE playlist_id=?", (playlist_id,)) @@ -11678,10 +11996,33 @@ class MusicDatabase: # ===================== HiFi Instances ===================== + def _ensure_hifi_instances_table(self, cursor) -> None: + """Defensive lazy-create. Issue #503: some users hit a "no such + table: hifi_instances" error when adding a HiFi instance even + though ``_initialize_database`` runs ``CREATE TABLE IF NOT EXISTS`` + on every boot. Root cause: the bulk init runs every CREATE + + every migration inside one transaction, so if any later migration + step throws on the user's specific DB shape, the whole batch + rolls back (Python's sqlite3 module doesn't autocommit DDL by + default) and ``hifi_instances`` never lands. This helper ensures + the table exists immediately before every operation that touches + it — idempotent, costs one PRAGMA-level no-op when the table is + already present, and fully recovers from a broken init.""" + cursor.execute(""" + CREATE TABLE IF NOT EXISTS hifi_instances ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT NOT NULL UNIQUE, + priority INTEGER NOT NULL DEFAULT 0, + enabled INTEGER NOT NULL DEFAULT 1, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + def get_hifi_instances(self) -> List[Dict[str, Any]]: """Get all enabled HiFi instances ordered by priority.""" conn = self._get_connection() cursor = conn.cursor() + self._ensure_hifi_instances_table(cursor) cursor.execute("SELECT url, priority, enabled FROM hifi_instances WHERE enabled = 1 ORDER BY priority ASC, id ASC") return [dict(row) for row in cursor.fetchall()] @@ -11689,6 +12030,7 @@ class MusicDatabase: """Get all HiFi instances (including disabled) ordered by priority.""" conn = self._get_connection() cursor = conn.cursor() + self._ensure_hifi_instances_table(cursor) cursor.execute("SELECT url, priority, enabled FROM hifi_instances ORDER BY priority ASC, id ASC") return [dict(row) for row in cursor.fetchall()] @@ -11696,6 +12038,7 @@ class MusicDatabase: """Add a new HiFi instance.""" conn = self._get_connection() cursor = conn.cursor() + self._ensure_hifi_instances_table(cursor) cursor.execute( "INSERT OR IGNORE INTO hifi_instances (url, priority, enabled) VALUES (?, ?, 1)", (url, priority) @@ -11707,6 +12050,7 @@ class MusicDatabase: """Remove a HiFi instance.""" conn = self._get_connection() cursor = conn.cursor() + self._ensure_hifi_instances_table(cursor) cursor.execute("DELETE FROM hifi_instances WHERE url = ?", (url,)) conn.commit() return cursor.rowcount > 0 @@ -11715,6 +12059,7 @@ class MusicDatabase: """Enable or disable a HiFi instance.""" conn = self._get_connection() cursor = conn.cursor() + self._ensure_hifi_instances_table(cursor) cursor.execute("UPDATE hifi_instances SET enabled = ? WHERE url = ?", (1 if enabled else 0, url)) conn.commit() return cursor.rowcount > 0 @@ -11727,6 +12072,7 @@ class MusicDatabase: return True conn = self._get_connection() cursor = conn.cursor() + self._ensure_hifi_instances_table(cursor) placeholders = ",".join("?" for _ in urls) cursor.execute( f"SELECT url FROM hifi_instances WHERE url IN ({placeholders})", @@ -11745,6 +12091,7 @@ class MusicDatabase: """Insert default instances if the table is empty.""" conn = self._get_connection() cursor = conn.cursor() + self._ensure_hifi_instances_table(cursor) cursor.execute("SELECT COUNT(*) as cnt FROM hifi_instances") count = cursor.fetchone()['cnt'] if count == 0: @@ -11790,5 +12137,5 @@ def close_database(): db_instance.close() except Exception as e: # Ignore threading errors during shutdown - pass + logger.debug("db instance close: %s", e) _database_instances.clear() diff --git a/docs/download-engine-refactor-plan.md b/docs/download-engine-refactor-plan.md new file mode 100644 index 00000000..92605783 --- /dev/null +++ b/docs/download-engine-refactor-plan.md @@ -0,0 +1,207 @@ +# Download Engine Refactor Plan + +## Goal + +Mirror Cin's "metadata engine" architecture for the download dispatcher. Move shared logic OUT of the per-source clients (currently 1600+ LOC of duplicated thread workers, search retry ladders, rate-limiters, state machines) and INTO a central `DownloadEngine`. Clients become dumb: make raw API requests + manage their own auth state. Everything else is the engine. + +This is the SAME architectural smell Cin flagged on the metadata layer, applied to downloads. If we keep adding sources (usenet planned + likely more), the only honest fix is to stop reinventing the wheel per client. + +## Architecture target + +``` +┌─────────────┐ ┌──────────────────┐ +│ feature │ ── search/download ──▶ ┌────────────────────┐ ─▶│ Soulseek (raw) │ +│ │ ◀── normalized ──────── │ DownloadEngine │ ─▶│ YouTube (raw) │ +└─────────────┘ │ │ ─▶│ Tidal (raw) │ + │ ◆ thread workers │ ─▶│ Qobuz (raw) │ + │ ◆ rate-limit pool │ ─▶│ HiFi (raw) │ + │ ◆ search retry │ ─▶│ Deezer (raw) │ + │ ◆ quality filter │ ─▶│ SoundCloud (raw) │ + │ ◆ state tracking │ ─▶│ Lidarr (album) │ + │ ◆ fallback chain │ └──────────────────┘ + │ ◆ cache │ clients only do: + └────────────────────┘ - raw API request + - auth/token state +``` + +## What clients keep (legitimately per-source) + +- Auth flow + token refresh (Tidal OAuth, Qobuz session, Deezer ARL, slskd API key, etc.) +- Source-specific protocol (slskd events vs HTTP REST vs HLS demux vs Blowfish decrypt vs yt-dlp subprocess) +- Source-specific search query shape (free text vs keyword filters vs MusicBrainz ID lookup) +- Source-specific "download a thing" atomic operation (`_download_impl(target_id) → file_path`) + +## What moves into the engine + +| Today (per-client, duplicated) | Tomorrow (engine, single source of truth) | +|---|---| +| `self.active_downloads = {}` per client | `engine.active_downloads = {}` | +| `self._download_lock = Lock()` per client | `engine.state_lock = Lock()` | +| `self._download_semaphore = Semaphore(...)` per client | `engine.download_pool` (per-source semaphore from registry) | +| `self._last_download_time / _download_delay` per client | `engine.rate_limiter.acquire(source)` | +| `_download_thread_worker` × 7 (~70 LOC each) | `engine.dispatch_download(plugin, target_id)` | +| Search retry ladder × 7 | `engine.search(query)` with shared retry policy | +| Quality filter × 7 | `engine.filter_by_quality(results, prefs)` | +| Result dedup × 7 | `engine.dedup(results)` | +| Hybrid fallback (search only) | `engine.fallback_chain(operation)` (search AND download) | + +## New plugin contract (much smaller) + +```python +class DownloadSourcePlugin(Protocol): + # Identity + name: str + + # Lifecycle + def is_configured(self) -> bool: ... + async def check_connection(self) -> bool: ... + def reload_settings(self) -> None: ... + + # Search — DUMB. Just hit the API. + async def search_raw(self, query: str) -> List[RawSearchResult]: ... + + # Download — DUMB. Just download the bytes to a file. + # The engine handles thread spawning, state tracking, rate limits. + # Plugin returns the final file path on success or raises. + async def download_raw(self, target_id: str, dest_dir: Path) -> Path: ... + + # Cancel — best-effort. Engine handles state cleanup. + async def cancel_raw(self, target_id: str) -> bool: ... +``` + +Compare to today's plugin protocol (which my Phase 0 PR introduced) — that one was wrapped around fat clients. This one is dumber. Clients shrink dramatically (estimated 40-60% LOC reduction per file). + +## Special cases + +- **Soulseek** — slskd is event-driven, NOT thread-based. Engine's BackgroundDownloadWorker doesn't apply. Keep Soulseek's path special: `download_raw` returns immediately; engine subscribes to slskd events for state updates instead of running a thread. +- **YouTube/SoundCloud** — yt-dlp is a subprocess. The "thread" is really `subprocess.run(['yt-dlp', ...]).wait()`. Engine handles thread; plugin's `download_raw` just runs subprocess and returns file path. +- **Lidarr** — album-grabber, not track-grabber. Different contract. Either separate `AlbumOnlyPlugin` interface OR plugin declares `supports_track_search: bool = False`. Decide during migration. + +## Phased commit plan + +Each phase is one or more commits. Each commit independently revertable. Tests stay green between commits — never ship a half-broken state. + +### Phase A — Behavior pinning tests (BEFORE any code moves) + +**Goal:** Baseline tests for what each source's download path currently does. Catches regressions during extraction. + +**Commit A1:** `tests/downloads/test_soulseek_download_path.py` — pin Soulseek's download lifecycle (search → download → completion → file path returned). +**Commit A2:** Same for YouTube. Mock yt-dlp subprocess. +**Commit A3:** Same for Tidal. Mock tidalapi.Session. +**Commit A4:** Same for Qobuz. Mock Qobuz REST API. +**Commit A5:** Same for HiFi. Mock hifi-api instance. +**Commit A6:** Same for Deezer. Mock Deezer GW API + Blowfish stream. +**Commit A7:** Same for SoundCloud. Mock yt-dlp scsearch. +**Commit A8:** Same for Lidarr. Mock Lidarr REST API. + +After Phase A: ~50 new tests pinning current behavior. We can refactor with confidence. + +### Phase B — Engine skeleton + state lift + +**Commit B1:** Create `core/download_engine/` package with `DownloadEngine` class. Engine starts EMPTY — just exposes `register_plugin(plugin)`, `active_downloads` dict, `state_lock`. Orchestrator gets a `self.engine` reference but doesn't use it yet. + +**Commit B2:** Move `active_downloads` state out of every client into `engine.active_downloads`. Each client's `download()` now updates engine state via callback instead of `self.active_downloads[id] = ...`. Backward compat: each client's `self.active_downloads` becomes a property that delegates to `engine.active_downloads.filter(source=self.name)`. + +**Commit B3:** Move `get_all_downloads` / `get_download_status` / `cancel_download` dispatch from orchestrator (which iterates plugins) into engine (which queries unified state). Orchestrator's methods become thin pass-throughs. + +### Phase C — Background download worker lift + +**Commit C1:** New `core/download_engine/worker.py` — `BackgroundDownloadWorker` class. Owns semaphore, rate-limit sleep, state-update lock pattern. Provides `dispatch(plugin, target_id, display_name) → download_id`. + +**Commit C2:** Migrate YouTube to use BackgroundDownloadWorker. Strip `_download_thread_worker` from `youtube_client.py`. Add `download_raw(video_id, dest) → Path`. Tests stay green (Phase A pinned them). + +**Commit C3:** Same for Tidal. +**Commit C4:** Same for Qobuz. +**Commit C5:** Same for HiFi. +**Commit C6:** Same for Deezer. +**Commit C7:** Same for SoundCloud. + +After Phase C: ~490 LOC of duplicated thread management deleted. Each affected client shrinks. + +### Phase D — SKIPPED + +**Original intent:** Lift search retry / query normalization / quality filter into engine. **Dropped after surveying actual per-source search code.** Search is 90% source-specific (slskd event subscription vs yt-dlp subprocess vs HTTP REST vs HLS quality map), not 60% like the original plan estimated. Lifting would be either lossy (force per-source quirks through a uniform interface) or bloated (adapter code bigger than the original). The shared portion is ~10 LOC per source — not worth a SearchOrchestrator. Per-source search stays per-source. + +### Phase D (original — kept for reference, NOT executed) + +**Commit D1:** New `core/download_engine/search.py` — `SearchOrchestrator`. Owns: query normalization, shortened-query retry ladder, quality filter, dedup. Calls `plugin.search_raw(query)` for the actual API hit. + +**Commit D2:** Migrate Tidal's search. Strip `_generate_shortened_queries`, quality filter, dedup from client. Add `search_raw(query) → List[RawResult]`. +**Commit D3:** Same for Qobuz. +**Commit D4:** Same for HiFi. +**Commit D5:** Same for YouTube. +**Commit D6:** Same for Deezer. +**Commit D7:** Same for SoundCloud. +**Commit D8:** Same for Soulseek (keep slskd event-driven specifics, but the post-search filter/dedup moves out). +**Commit D9:** Same for Lidarr. + +### Phase E — Rate-limit pool + +**Commit E1:** New `core/download_engine/rate_limit.py` — per-source rate limiter registry. Spotify limit, Qobuz 1/sec, etc. Each plugin declares its limits in its registry spec. +**Commit E2:** Strip per-client rate-limit state. Replace with `await engine.rate_limit.acquire(self.name)` at the top of `search_raw` / `download_raw`. + +### Phase F — Fallback chain into engine + +**Commit F1:** Engine owns fallback: `engine.search_with_fallback(query, source_chain)` and `engine.download_with_fallback(target_id, source_chain)`. Search hybrid behavior preserved; download hybrid newly works (today it silently routes to one source). +**Commit F2:** Orchestrator's `search` and `download` methods delegate to engine's fallback methods. Hybrid mode logic moves out of orchestrator. + +### Phase G — Plugin contract narrows + +**Commit G1:** Update `DownloadSourcePlugin` Protocol — narrow to the small surface (`search_raw`, `download_raw`, `cancel_raw`, `is_configured`, `check_connection`, `reload_settings`). Conformance tests updated. +**Commit G2:** Remove dead methods from clients that the engine now owns (`_download_thread_worker`, `_filter_results_by_quality`, etc.). Clean up imports. + +### Phase H — Cleanup + WHATS_NEW + version bump + +**Commit H1:** Final cleanup pass — remove backward-compat shims that are no longer needed (legacy `self.active_downloads` properties etc., once nothing reaches in for them). +**Commit H2:** WHATS_NEW entry, PR description. + +## Total estimated scope + +- ~25-30 commits +- ~2000 LOC removed (duplicated thread workers, search retries, etc.) +- ~1200 LOC added (engine + per-source slim adapters) +- Net reduction: ~800 LOC +- ~50 new tests (Phase A pinning) + ~20 engine-level tests +- 1-2 days of focused work + +## Risk profile + +**Low risk:** +- Phase A (only adds tests, never changes behavior) +- Phase B1 (new file, doesn't touch existing code) +- Phase H (cleanup of dead code) + +**Medium risk:** +- Phase B2-B3 (state lift — race conditions, lock contention) +- Phase C (thread worker extraction — semaphore semantics, exception propagation) +- Phase G (contract narrows — anything reaching in for removed methods breaks) + +**High risk:** +- Phase D (search retry — easy to subtly change retry ladder shape) +- Phase E (rate-limit — wrong order can cause deadlocks or under-limit violations) +- Phase F (fallback — easy to accidentally change hybrid mode behavior) + +**Mitigation:** Phase A pinning tests catch behavior drift in every later phase. Each commit must pass full suite. Manual smoke test per source after Phase C and again at end. + +## Coordination with Cin + +- Cin's metadata engine PR will likely set the precedent for HOW abstractions look (Protocol vs ABC, sync vs async, state location). This plan defaults to Protocol + async (matches what we already have) but easy to mirror Cin's exact pattern when his PR lands. +- If his pattern differs significantly, we may need to redo some commits. Best mitigation: don't dig too deep on contract shape (Phase G) until his PR is visible. Phases A-C don't depend on contract shape; they're safe to do regardless. +- Send Cin a heads-up before starting — he may have feedback on the plan that saves a redesign later. + +## Compatibility commitments + +- **Originally** the plan was to preserve `orchestrator.soulseek` / `orchestrator.youtube` / etc. attribute aliases through every phase. Cin's review pass removed them in favor of the generic `orchestrator.client('')` accessor — this is a breaking change for any external code that reached into per-source attributes directly. Anything in-tree was migrated as part of the same PR; in-flight branches will need to update. +- The legacy `soulseek_client` global handle was renamed to `download_orchestrator` in the same review pass. Any module that imported / referenced `soulseek_client` was migrated; in-flight branches will need to update. +- Soulseek-specific helper methods (`clear_all_searches`, `_make_request`, `signal_download_completion`, etc.) still live on the orchestrator and continue to work — but reach the underlying SoulseekClient via `orchestrator.client('soulseek')` instead of `orchestrator.soulseek`. +- Frontend status dashboard keys (`deezer_dl` alias) preserved. +- Config format unchanged. +- DB schema unchanged. +- API endpoint surface unchanged. + +## What's NOT in this PR + +- Cin's metadata engine work (separate, his domain) +- Media server client refactor (different subsystem, separate PR) +- Match engine refactor (different subsystem, separate PR) +- Adding new download sources (out of scope; the new contract makes them easier later) diff --git a/docs/media-server-engine-refactor-plan.md b/docs/media-server-engine-refactor-plan.md new file mode 100644 index 00000000..9f271f66 --- /dev/null +++ b/docs/media-server-engine-refactor-plan.md @@ -0,0 +1,159 @@ +# Media Server Engine Refactor Plan + +## Goal + +Same playbook as the download engine refactor, applied to media servers. Replace 33 `if active_server == 'plex' / 'jellyfin' / 'navidrome' / 'soulsync'` dispatch sites in web_server.py with a central `MediaServerEngine` that owns server selection + cross-server query dispatch. Each per-server client stays as-is for its protocol-specific work (Plex API SDK, Jellyfin REST, Navidrome OpenSubsonic, SoulSync filesystem walk). + +**The smell:** Adding a 5th server (recent SoulSync standalone showed how) requires hunting through 33 web_server.py dispatch sites + DatabaseUpdateWorker branching + UI sync-button visibility hacks. Plex assumptions had leaked into nominally generic code paths. + +## Architecture target + +``` +┌───────────┐ ┌──────────────────┐ +│ feature │ ─search/scan/sync─▶ ┌──│ MediaServerEngine │ ─▶ Plex (PlexAPI SDK) +│ │ ◀── results ────── │ │ ◆ active selection│ ─▶ Jellyfin (REST) +└───────────┘ │ │ ◆ dispatch │ ─▶ Navidrome (Subsonic) + │ │ ◆ shared shape │ ─▶ SoulSync (filesystem) + │ └──────────────────┘ + │ (no need for per-server thread pool — server APIs + │ are sync-call shaped, not stream-shaped like downloads) +``` + +## What clients keep (per-server, legitimately) + +- **Auth** — Plex token, Jellyfin API key + user ID + library ID, Navidrome user/pass + salt, SoulSync filesystem path +- **Wire protocol** — PlexAPI SDK objects vs Jellyfin REST `/Items` vs Navidrome XML/JSON Subsonic vs filesystem walk +- **ID schemes** — Plex ratingKey (int), Jellyfin GUID (hex), Navidrome string, SoulSync MD5 +- **Connection state** — each client owns its session / token / connection object +- **Cache strategy** — Jellyfin's aggressive pre-cache, Navidrome's folder filter, SoulSync's 5-min TTL +- **Wrapper objects** — `JellyfinTrack`, `NavidromeTrack`, etc. stay in client modules + +## What moves into the engine + +| Today (web_server.py duplicated) | Tomorrow (MediaServerEngine, single dispatch site) | +|---|---| +| `if active_server == 'plex': plex_client.X() elif 'jellyfin': jellyfin_client.X() ...` | `engine.X()` — engine reads `active_server` once, routes | +| Status check 4-way branch | `engine.is_connected()` | +| Library scan trigger 3-way | `engine.trigger_library_scan()` | +| Search dispatch in 3 sites | `engine.search_tracks(title, artist)` | +| Play history 4-way | `engine.get_play_history()` | +| Metadata writeback 4-way (genre / poster / bio) | `engine.update_artist_genres()` etc. — engine routes, plugin no-ops where unsupported | +| `DatabaseUpdateWorker.server_type` branching | `engine` injected, no per-server branching | + +## Plugin contract (ABC, narrower than the download contract) + +```python +class MediaServerClient(ABC): + # Connection + @abstractmethod + def is_connected(self) -> bool: ... + @abstractmethod + def ensure_connection(self) -> bool: ... + + # Library reads (required) + @abstractmethod + def get_all_artists(self): ... + @abstractmethod + def get_all_album_ids(self): ... + @abstractmethod + def search_tracks(self, title, artist, limit=15): ... + @abstractmethod + def get_recently_added_albums(self, max_results=400): ... + + # Library writes (required for the scan endpoints — may no-op for SoulSync) + @abstractmethod + def trigger_library_scan(self) -> bool: ... + @abstractmethod + def is_library_scanning(self) -> bool: ... + + # Playlist sync (optional — Navidrome stub returns True) + def create_playlist(self, name, tracks) -> bool: return True + def update_playlist(self, name, tracks) -> bool: return True + def copy_playlist(self, source, dest_name) -> bool: return True + + # Analytics (optional) + def get_play_history(self, limit=500) -> list: return [] + def get_track_play_counts(self) -> dict: return {} + + # Metadata writeback (optional — clients no-op where API doesn't support it) + def update_artist_genres(self, artist, genres) -> bool: return True + def update_artist_poster(self, artist, image_data) -> bool: return True + def update_album_poster(self, album, image_data) -> bool: return True + def update_artist_biography(self, artist) -> bool: return True +``` + +## Phased commit plan + +### Phase 0 — Foundation +- **0.1** Plugin contract (`core/media_server/contract.py`) + ABC +- **0.2** Registry + dispatch helpers (`core/media_server/registry.py`) +- **0.3** Conformance tests — every registered client satisfies the ABC + +### Phase A — Behavior pinning +- **A1** Pin Plex client surface (status check, search, scan trigger, metadata writeback shape) +- **A2** Pin Jellyfin client surface +- **A3** Pin Navidrome client surface +- **A4** Pin SoulSync standalone client surface + +### Phase B — Engine skeleton +- **B1** `MediaServerEngine` skeleton — holds clients, exposes `active_server` selection +- **B2** Engine dispatch methods (`is_connected`, `search_tracks`, `trigger_library_scan`, `is_library_scanning`, etc.) +- **B3** Engine-level conformance tests + +### Phase C — Migrate web_server.py dispatch sites +Each commit migrates a logical cluster (status, scan, playlist, metadata, history) so the diff per commit is small + reviewable. + +- **C1** Status / connection-check sites +- **C2** Library scan trigger + scanning-state polling +- **C3** Search dispatch (3 sites) +- **C4** Playlist sync / apply / suggest +- **C5** Play history + track play counts +- **C6** Metadata writeback (genres / posters / bio) + +### Phase D — DatabaseUpdateWorker +- **D1** Inject engine into `DatabaseUpdateWorker` +- **D2** Strip `self.server_type` branching (use `engine.get_active_client_type()` where shape genuinely differs) + +### Phase E — Cleanup + ship +- **E1** Drop unused imports / dead code +- **E2** WHATS_NEW + PR description + +**Total estimated:** 15-18 commits. ~600 LOC moved, ~200 LOC deleted. + +## Risk profile + +**Low:** +- Phase 0 (pure additive — new contract, no existing code touched) +- Phase A (tests only) +- Phase B (engine exists but nothing routes through it yet) + +**Medium:** +- Phase C (each commit changes 5-10 dispatch sites in web_server.py — cross-cutting) +- Phase D (DatabaseUpdateWorker is a hot path during library refresh) + +**Mitigation:** +- Phase A pinning catches per-client behavior drift +- Phase B engine tested in isolation before C wires it in +- Phase C commits are small + reversible (one cluster per commit) +- Suite green at every commit + +## What's NOT in this PR + +- Unifying track ID schemes (Plex int vs Jellyfin GUID vs Navidrome string vs SoulSync MD5) — separate data model refactor +- Merging Plex/Jellyfin/Navidrome wrapper objects (each tied to its API) +- Extracting common metadata writeback logic — too server-specific +- Adding new server types (Subsonic, MusicBrainz local) — out of scope; this PR makes them easier later + +## Coordination with other work + +- Independent of the download engine refactor (PR #494) — different subsystem +- Cin's metadata engine work also independent +- No collisions expected + +## Compatibility commitments + +- All existing config keys preserved +- All existing API endpoints preserved +- Plex/Jellyfin/Navidrome/SoulSync clients keep their public methods (engine wraps, doesn't replace) +- Active-server config (`server.active`) unchanged +- `DatabaseUpdateWorker.server_type` available for external readers (deprecated but not removed) diff --git a/docs/metadata-types-migration.md b/docs/metadata-types-migration.md new file mode 100644 index 00000000..393e6c09 --- /dev/null +++ b/docs/metadata-types-migration.md @@ -0,0 +1,125 @@ +# Typed Metadata Migration Plan + +## Why + +Right now the metadata pipeline has no real contract about the shape +of data flowing between providers and consumers. Each provider +(Spotify, iTunes, Deezer, Tidal, Qobuz, MusicBrainz, AudioDB, +Discogs, Hydrabase) returns its own response shape, and consumer +code defensively extracts every field via fallback chains: + +```python +def _build_album_info(album_data, album_id, album_name='', artist_name=''): + images = _extract_lookup_value(album_data, 'images', default=[]) or [] + ... + return { + 'id': _extract_lookup_value(album_data, 'id', 'album_id', + 'collectionId', 'release_id', + default=album_id) or album_id, + ... + } +``` + +This pattern works but makes the codebase hard to extend safely: + +- Adding a new provider means adding more keys to the fallback chains + in every consumer file (currently ~150 call sites of + `_extract_lookup_value` across the codebase). +- Fixing a bug in extraction means fixing it in N places. +- New consumers can't trust the data — they re-run defensive logic. +- Tests are theatre because the contract is "whatever shape happens + to come in." + +## What this PR adds + +`core/metadata/types.py` defines the canonical typed dataclasses: + +- `Album` — required fields: `id`, `name`, `artists`, `release_date`, + `total_tracks`, `album_type`. Optional: `image_url`, `artist_id`, + `genres`, `label`, `barcode`, `external_ids`, `external_urls`. +- `Track` — required fields: `id`, `name`, `artists`, `album`, + `duration_ms`. Optional: track/disc number, image, ISRC, etc. +- `Artist` — required fields: `id`, `name`. Optional: image, genres. + +Plus per-provider classmethod converters on `Album`: + +- `Album.from_spotify_dict(raw)` +- `Album.from_itunes_dict(raw)` +- `Album.from_deezer_dict(raw)` +- `Album.from_discogs_dict(raw)` +- `Album.from_musicbrainz_dict(raw)` +- `Album.from_hydrabase_dict(raw)` +- `Album.from_qobuz_dict(raw)` +- `Album.from_tidal_object(obj)` — note: Tidal goes through the + ``tidalapi`` library which returns Python objects rather than + raw dicts, so this converter is named ``_object`` not ``_dict`` + to make the input contract explicit. + +Enrichment-only providers (Last.fm, Genius, AcoustID, ListenBrainz, +AudioDB) don't return Album-shaped responses — they enrich +existing rows with tags, lyrics URLs, fingerprint matches, etc. +No Album converter needed for those. + +Each converter is the SINGLE place that knows that provider's wire +shape. Adding a new provider = adding one classmethod here and +nothing else needs to change. + +`Album.to_context_dict()` returns the canonical dict shape SoulSync's +existing import / download pipelines expect — the bridge between +typed data and the current dict-passing internal API. + +## What this PR DOES NOT do + +This PR does not migrate any consumer. No behavior changes. The new +types and converters are pure additive — every existing code path +keeps using `_extract_lookup_value` exactly as before. + +The reason: a single big-bang migration would be a 153-call-site +refactor with subtle behavior risk. Better to land the foundation +in isolation, prove the contract via tests, then migrate consumers +one at a time in follow-up PRs that are individually reviewable +and revertable. + +## Migration roadmap + +Numbered in suggested order. Each item is its own PR. + +1. **Foundation (this PR).** Land `core/metadata/types.py` + + converters + tests. Document migration plan. +2. **Migrate `_build_album_info`** in + `core/metadata/album_tracks.py` — accept either a typed `Album` + OR a raw dict. When it gets a typed Album, return + `album.to_context_dict()`. When it gets a raw dict, normalize + via the appropriate `from__dict()` based on the + provided `source` argument. Reduces from 41 LOC of fallback + chains to ~5 LOC of dispatch. +3. **Migrate `_build_single_import_context_payload`** in the same + file — same pattern. +4. **Migrate Spotify client.** `SpotifyClient.get_album()` returns + `Album` instead of raw dict. Internal callers update. Public + API surface unchanged where it has to be (return both for one + release, deprecate dict version). +5. **Migrate iTunes/Deezer/Tidal/Qobuz/Discogs/Hydrabase clients.** + Same pattern. Each client's `get_album()` returns `Album`. +6. **Migrate consumers in `core/discovery/quality_scanner.py`, + `core/imports/context.py`, etc.** Drop their fallback chains + in favor of typed access. +7. **Add `Track` converters and migrate Track-shaped consumers.** + Same pattern as Album. +8. **Add `Artist` converters and migrate Artist-shaped consumers.** +9. **Deprecate `_extract_lookup_value`.** Once no caller needs it, + delete it. + +Each PR is independently revertable. Behavior preserved at every +step. + +## Acceptance criteria for this PR + +- All converters produce a fully-populated `Album` from realistic + provider response samples. +- Every required field is set even when source data is partial. +- `to_context_dict()` shape is identical across all six providers + (pinned via cross-provider parametrized tests). +- No existing consumer is changed; existing tests pass unchanged. +- Cross-provider invariants (release_date format, album_type values, + Discogs `(N)` stripping, iTunes artwork upgrade) are pinned. diff --git a/entrypoint.sh b/entrypoint.sh index 1c9f1382..8bbf43d8 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -35,9 +35,15 @@ if [ "$CURRENT_UID" != "$PUID" ] || [ "$CURRENT_GID" != "$PGID" ]; then usermod -o -u "$PUID" soulsync fi - # Fix ownership of app directories - echo "🔒 Fixing permissions on app directories..." - chown -R soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging 2>/dev/null || true + # Only do the expensive recursive chown if the data directory ownership + # doesn't already match — avoids walking large libraries on every restart. + DATA_OWNER=$(stat -c '%u:%g' /app/data 2>/dev/null || echo "unknown") + if [ "$DATA_OWNER" != "$PUID:$PGID" ]; then + echo "🔒 Fixing permissions on app directories..." + chown -R soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging 2>/dev/null || true + else + echo "✅ App directory permissions already correct" + fi else echo "✅ User/Group IDs already correct" fi @@ -63,9 +69,11 @@ echo " 📄 Updating settings.py to current version..." cp /defaults/settings.py /app/config/settings.py chown soulsync:soulsync /app/config/settings.py 2>/dev/null || true -# Ensure all directories exist and have proper permissions +# Ensure all directories exist with correct ownership. +# Only the directory nodes themselves need chown here — the recursive chown +# above already ran if UIDs changed, so avoid walking the whole tree every start. mkdir -p /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging -chown -R soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging 2>/dev/null || true +chown soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging 2>/dev/null || true echo "✅ Configuration initialized successfully" diff --git a/pyproject.toml b/pyproject.toml index 79ca43ed..f82a69fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ line-length = 160 [tool.ruff.lint] # Start lenient — catch real bugs, not style nits # E: pycodestyle errors, F: pyflakes, UP: pyupgrade, B: bugbear -select = ["F", "E9", "W6", "B"] +select = ["F", "E9", "W6", "B", "S110"] # Ignore rules that will flag too much in an existing codebase ignore = [ "F401", # unused imports (too noisy initially) @@ -15,4 +15,9 @@ ignore = [ [tool.ruff.lint.per-file-ignores] # Tests can use assert, magic values, etc. -"tests/**" = ["B", "F"] +"tests/**" = ["B", "F", "S110"] + +[tool.pytest.ini_options] +markers = [ + "soundcloud_live: live SoundCloud integration test (network-dependent, run with -m soundcloud_live)", +] diff --git a/requirements-dev.txt b/requirements-dev.txt index 587022df..af02d432 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # Runtime web dependencies + test runner -r requirements.txt -ruff +ruff==0.15.12 # Test runner -pytest>=9.0.0 +pytest==9.0.3 diff --git a/requirements.txt b/requirements.txt index 16d2b801..73ef28a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,48 +1,49 @@ # SoulSync requirements # Web application dependencies only +# All dependencies pinned for reproducible builds. # Core web framework -Flask>=3.0.0 -Flask-Limiter>=3.5.0 +Flask==3.1.3 +Flask-Limiter==4.1.1 # Music service APIs -spotipy>=2.23.0 -PlexAPI>=4.17.0 +spotipy==2.26.0 +PlexAPI==4.18.1 -# HTTP and async support -requests>=2.31.0 -aiohttp>=3.9.0 +# HTTP and async support +requests==2.33.1 +aiohttp==3.13.5 # Security and encryption -cryptography>=41.0.0 +cryptography==48.0.0 # Media metadata handling -mutagen>=1.47.0 -Pillow>=10.0.0 +mutagen==1.47.0 +Pillow==12.2.0 # Text processing -unidecode>=1.3.8 -beautifulsoup4>=4.12.0 +Unidecode==1.4.0 +beautifulsoup4==4.14.3 # System monitoring -psutil>=6.0.0 +psutil==7.2.2 -# YouTube support — pinned for reproducible builds; bump per release. See #367. -yt-dlp==2026.3.17 +# YouTube support -- unpinned; yt-dlp must track upstream releases to stay functional +yt-dlp>=2026.3.17 # Lyrics support -lrclibapi>=0.3.1 +lrclibapi==0.3.1 # Audio fingerprinting for download verification -pyacoustid>=1.3.0 +pyacoustid==1.3.1 # WebSocket client for Hydrabase connection -websocket-client>=1.7.0 +websocket-client==1.9.0 # Tidal download support -tidalapi>=0.7.6 +tidalapi==0.8.11 # WebSocket server for real-time UI updates -flask-socketio>=5.3.0 -gunicorn>=25.3.0 -simple-websocket>=1.1.0 +flask-socketio==5.6.1 +gunicorn==26.0.0 +simple-websocket==1.1.0 diff --git a/services/sync_service.py b/services/sync_service.py index 4582a5cf..86b12920 100644 --- a/services/sync_service.py +++ b/services/sync_service.py @@ -4,10 +4,8 @@ from dataclasses import dataclass from datetime import datetime from utils.logging_config import get_logger from core.spotify_client import SpotifyClient, Playlist as SpotifyPlaylist, Track as SpotifyTrack -from core.plex_client import PlexClient, PlexTrackInfo -from core.jellyfin_client import JellyfinClient -from core.navidrome_client import NavidromeClient -from core.soulseek_client import SoulseekClient +from core.media_server.types import TrackInfo +from core.download_orchestrator import DownloadOrchestrator from core.matching_engine import MusicMatchingEngine, MatchResult logger = get_logger("sync_service") @@ -44,17 +42,35 @@ class SyncProgress: failed_tracks: int = 0 class PlaylistSyncService: - def __init__(self, spotify_client: SpotifyClient, plex_client: PlexClient, soulseek_client: SoulseekClient, jellyfin_client: JellyfinClient = None, navidrome_client = None): + def __init__(self, spotify_client: SpotifyClient, download_orchestrator: DownloadOrchestrator, media_server_engine=None): + """Initialize the sync service. + + ``media_server_engine`` is the central MediaServerEngine that owns + the per-server clients (Plex / Jellyfin / Navidrome / SoulSync). + Replaces the legacy per-server kwargs (plex_client / jellyfin_client + / navidrome_client) — all media-server access now goes through + ``self._engine.client(name)`` so swapping the active server doesn't + need a service rebuild. + + ``download_orchestrator`` (formerly ``soulseek_client``) is the + DownloadOrchestrator that owns the per-source download clients — + rename + retype landed via the download refactor PR. + """ self.spotify_client = spotify_client - self.plex_client = plex_client - self.jellyfin_client = jellyfin_client - self.navidrome_client = navidrome_client - self.soulseek_client = soulseek_client + self._engine = media_server_engine + self.download_orchestrator = download_orchestrator self.progress_callbacks = {} # Playlist-specific progress callbacks self.syncing_playlists = set() # Track multiple syncing playlists self._cancelled = False self.matching_engine = MusicMatchingEngine() - + + def _media_client(self, name: str): + """Resolve a per-server client through the engine, or None when the + engine isn't wired (defensive — every production path passes one).""" + if self._engine is None: + return None + return self._engine.client(name) + def _get_active_media_client(self, profile_id=None): """Get the active media client based on config settings. @@ -68,26 +84,27 @@ class PlaylistSyncService: active_server = config_manager.get_active_media_server() if active_server == "jellyfin": - if not self.jellyfin_client: + client = self._media_client('jellyfin') + if not client: logger.error("Jellyfin client not provided to sync service") return None, "jellyfin" - # Apply per-profile Jellyfin library if set if profile_id: - self._apply_profile_library(profile_id, 'jellyfin', self.jellyfin_client) - return self.jellyfin_client, "jellyfin" + self._apply_profile_library(profile_id, 'jellyfin', client) + return client, "jellyfin" elif active_server == "navidrome": - if not self.navidrome_client: + client = self._media_client('navidrome') + if not client: logger.error("Navidrome client not provided to sync service") return None, "navidrome" - return self.navidrome_client, "navidrome" + return client, "navidrome" else: # Default to Plex - # Apply per-profile Plex library if set - if profile_id: - self._apply_profile_library(profile_id, 'plex', self.plex_client) - return self.plex_client, "plex" + client = self._media_client('plex') + if profile_id and client: + self._apply_profile_library(profile_id, 'plex', client) + return client, "plex" except Exception as e: logger.error(f"Error determining active media server: {e}") - return self.plex_client, "plex" # Fallback to Plex + return self._media_client('plex'), "plex" # Fallback to Plex def _apply_profile_library(self, profile_id, server_type, client): """Apply per-profile library selection to a media client if configured.""" @@ -441,7 +458,7 @@ class PlaylistSyncService: self.clear_progress_callback(playlist.name) self._cancelled = False - async def _find_track_in_media_server(self, spotify_track: SpotifyTrack) -> Tuple[Optional[PlexTrackInfo], float]: + async def _find_track_in_media_server(self, spotify_track: SpotifyTrack) -> Tuple[Optional[TrackInfo], float]: """Find a track using the same improved database matching as Download Missing Tracks modal""" try: # Check active media server connection @@ -528,8 +545,8 @@ class PlaylistSyncService: spotify_id, me.clean_title(original_title), me.clean_artist(artist_name), active_server, db_track.id, db_track.title, confidence ) - except Exception: - pass + except Exception as e: + logger.debug("save sync match cache failed: %s", e) # Fetch the actual track object from active media server using the database track ID try: @@ -635,7 +652,7 @@ class PlaylistSyncService: query = self.matching_engine.generate_download_query(match_result.spotify_track) logger.info(f"Attempting to download: {query}") - download_id = await self.soulseek_client.search_and_download_best(query, expected_track=match_result.spotify_track) + download_id = await self.download_orchestrator.search_and_download_best(query, expected_track=match_result.spotify_track) if download_id: downloaded_count += 1 diff --git a/tests/artists/test_quality.py b/tests/artists/test_quality.py index 6d7574e8..5e556f4f 100644 --- a/tests/artists/test_quality.py +++ b/tests/artists/test_quality.py @@ -91,22 +91,50 @@ def _build_deps( artist_detail=None, wishlist=None, fallback_client=None, + fallback_source=None, + search_sources=None, profile_id=1, quality_tier=('mp3_320', 4), ): + """Build deps for tests. + + ``search_sources`` is the new contract — list of ``(name, client)`` + pairs the multi-source search dispatches across. For convenience, + ``fallback_client`` + ``fallback_source`` are still supported and + auto-translate to a single-source list when ``search_sources`` + isn't passed (preserves the older test ergonomics). + """ + if search_sources is None: + # Default: build a single-source list from fallback_client + + # fallback_source, mirroring the legacy single-source contract. + if fallback_source is None: + fallback_source = 'spotify' if spotify is not None else 'itunes' + if fallback_client is None and fallback_source == 'spotify': + fallback_client = spotify + search_sources = ( + [(fallback_source, fallback_client)] if fallback_client else [] + ) + # Mirror web_server._resolve_search_sources: Spotify is always + # added to the source list when a Spotify client is configured, + # alongside whatever else the user has connected. Tests passing + # ``spotify=`` get Spotify in the source list automatically so + # the direct-lookup fast path can find it. + if spotify is not None and not any(name == 'spotify' for name, _ in search_sources): + search_sources = [('spotify', spotify)] + list(search_sources) deps = aq.ArtistQualityDeps( - spotify_client=spotify, matching_engine=matching_engine or _FakeMatchingEngine(), get_database=lambda: _FakeDatabase(artist_detail=artist_detail), get_wishlist_service=lambda: wishlist or _FakeWishlist(), get_current_profile_id=lambda: profile_id, get_quality_tier_from_extension=lambda fp: quality_tier, - get_metadata_fallback_client=lambda: fallback_client, + get_metadata_search_sources=lambda: list(search_sources), ) return deps -def _artist_with_track(*, track_id='t1', file_path='/file.mp3', spotify_tid=None): +def _artist_with_track(*, track_id='t1', file_path='/file.mp3', + spotify_tid=None, deezer_tid=None, itunes_tid=None, + soul_id=None): return { 'success': True, 'artist': {'name': 'Artist Name'}, @@ -118,6 +146,9 @@ def _artist_with_track(*, track_id='t1', file_path='/file.mp3', spotify_tid=None 'title': 'Track One', 'file_path': file_path, 'spotify_track_id': spotify_tid, + 'deezer_id': deezer_tid, + 'itunes_track_id': itunes_tid, + 'soul_id': soul_id, 'track_number': 1, 'duration': 180000, 'bitrate': 320, @@ -168,12 +199,17 @@ def test_spotify_direct_lookup_via_track_id_uses_raw_data(): def test_spotify_direct_lookup_enhanced_format_rebuilds_payload(): - """Track details without raw_data → rebuild payload with album images via get_album.""" - enhanced = {'name': 'Track One', 'artists': ['Artist Name'], + """Track details without raw_data → rebuild wishlist-shape payload + from the enhanced top-level fields. Spotify enhanced shape returns + ``artists`` as a list of strings; the converter normalizes to + Spotify's wishlist shape (``[{'name': ...}]``). Album images stay + empty when the source doesn't surface them on the enhanced dict — + the wishlist re-download fetches art at download time.""" + enhanced = {'id': 'sp-stored', 'name': 'Track One', + 'artists': ['Artist Name'], 'album': {'id': 'alb-id', 'name': 'Album X'}, 'duration_ms': 180000, 'track_number': 1, 'disc_number': 1} - full_album = {'images': [{'url': 'http://art'}]} - spotify = _FakeSpotify(track_details=enhanced, album=full_album) + spotify = _FakeSpotify(track_details=enhanced) wishlist = _FakeWishlist() deps = _build_deps( spotify=spotify, @@ -185,7 +221,12 @@ def test_spotify_direct_lookup_enhanced_format_rebuilds_payload(): assert payload['enhanced_count'] == 1 md = wishlist.added[0]['spotify_track_data'] - assert md['album']['images'] == [{'url': 'http://art'}] + assert md['id'] == 'sp-stored' + assert md['name'] == 'Track One' + # Enhanced format: artists normalized from [str] → [{'name': str}]. + assert md['artists'] == [{'name': 'Artist Name'}] + assert md['album']['name'] == 'Album X' + assert md['album']['artists'] == [{'name': 'Artist Name'}] # --------------------------------------------------------------------------- @@ -193,11 +234,15 @@ def test_spotify_direct_lookup_enhanced_format_rebuilds_payload(): # --------------------------------------------------------------------------- def test_spotify_search_fallback_when_no_stored_id(): - """No spotify_track_id → search via matching_engine, pick best match.""" - track = _SpotifyTrack(name='Track One', artists=['Artist Name']) - raw = {'id': 'sp-search', 'name': 'Track One', 'artists': [{'name': 'Artist Name'}], - 'album': {'name': 'Album X'}} - spotify = _FakeSpotify(track_details={'raw_data': raw}, search_results=[track]) + """No spotify_track_id → multi-source text search runs, builds + wishlist payload from the source-native Track object via + ``_build_payload_from_track`` (not ``get_track_details`` — search + results already carry enough Track-shape fields, no extra API call + needed).""" + track = _SpotifyTrack(id='sp-search', name='Track One', + artists=['Artist Name']) + track.album = 'Album X' + spotify = _FakeSpotify(search_results=[track]) wishlist = _FakeWishlist() deps = _build_deps( spotify=spotify, @@ -208,7 +253,11 @@ def test_spotify_search_fallback_when_no_stored_id(): payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) assert payload['enhanced_count'] == 1 - assert wishlist.added[0]['spotify_track_data'] == raw + md = wishlist.added[0]['spotify_track_data'] + assert md['id'] == 'sp-search' + assert md['name'] == 'Track One' + assert md['artists'] == [{'name': 'Artist Name'}] + assert md['album']['name'] == 'Album X' # --------------------------------------------------------------------------- @@ -240,6 +289,291 @@ def test_fallback_source_when_spotify_none(): assert md['album']['images'] == [{'url': 'http://it', 'height': 600, 'width': 600}] +def test_dispatches_through_primary_source_not_spotify_specific(): + """Architectural pin: when the user's primary metadata source is + Discogs (or any non-Spotify source), the enhance flow searches + THAT source's client, not Spotify. Pre-fix the flow had a + hardcoded Spotify-direct → Spotify-search → iTunes-fallback chain + that ignored the user's actual configured primary source. + """ + discogs_track = _SpotifyTrack(id='dc-1', name='Track One', + artists=['Artist Name']) + discogs_track.track_number = 1 + discogs_track.disc_number = 1 + discogs_track.album = 'Album X' + discogs_calls = [] + + class _DiscogsStub: + def search_tracks(self, q, limit=5): + discogs_calls.append(q) + return [discogs_track] + + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(), + wishlist=wishlist, + fallback_client=_DiscogsStub(), + fallback_source='discogs', + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + + # Discogs (the configured primary) was the one searched; match queued. + assert discogs_calls, "Primary source (Discogs) was not searched" + assert payload['enhanced_count'] == 1 + assert wishlist.added[0]['spotify_track_data']['id'] == 'dc-1' + + +def test_spotify_direct_lookup_runs_as_fast_path_then_falls_back(): + """Architectural pin: Spotify direct-lookup is a fast-path + optimization, NOT a primary-source gate. When the user has Spotify + configured AND a stored spotify_track_id, direct lookup runs first + regardless of which other sources are wired. If it returns nothing, + the multi-source parallel search runs and the cross-source best + match wins (in this test, Discogs). + + This pins the post-refactor behavior — pre-refactor direct lookup + was gated on Spotify being the active primary source, which broke + enhance for users without Spotify primary even when they had a + stored Spotify ID. + """ + spotify_calls = [] + + class _SpotifyStub: + def get_track_details(self, tid): + spotify_calls.append(('details', tid)) + return None + def search_tracks(self, q, limit=10): + spotify_calls.append(('search', q)) + return [] + + discogs_track = _SpotifyTrack(id='dc-1', name='Track One', + artists=['Artist Name']) + discogs_track.track_number = 1 + discogs_track.disc_number = 1 + discogs_track.album = 'Album X' + + class _DiscogsStub: + def search_tracks(self, q, limit=10): + return [discogs_track] + + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=_SpotifyStub(), + artist_detail=_artist_with_track(spotify_tid='sp-stored'), + wishlist=wishlist, + fallback_client=_DiscogsStub(), + fallback_source='discogs', + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + + # Fast path was attempted (Spotify configured + stored ID present). + assert ('details', 'sp-stored') in spotify_calls + # Fast path returned None → multi-source search ran → Discogs won. + assert payload['enhanced_count'] == 1 + assert wishlist.added[0]['spotify_track_data']['id'] == 'dc-1' + + +# --------------------------------------------------------------------------- +# Direct-lookup-by-stored-ID (priority 1) — applies to every source +# with a stored ID column, not just Spotify +# --------------------------------------------------------------------------- + +def test_direct_lookup_via_deezer_id_skips_text_search(): + """Library track has stored deezer_id + Deezer is configured → + enhance fast-paths via deezer_client.get_track_details(id) and skips + fuzzy text search entirely. Mirrors what Download Discography does + (stable IDs, no fuzzy matching). Pre-fix Deezer-primary users went + through text search even when the deezer_id was already on the row. + """ + deezer_calls = [] + enhanced_dict = { + 'id': '12345', 'name': 'Track One', + 'artists': ['Artist Name'], + 'album': {'id': 'alb-1', 'name': 'Album X'}, + 'duration_ms': 180000, 'track_number': 1, 'disc_number': 1, + } + + class _DeezerStub: + def get_track_details(self, tid): + deezer_calls.append(('details', tid)) + return enhanced_dict + def search_tracks(self, q, limit=10): + deezer_calls.append(('search', q)) + return [] + + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(deezer_tid='12345'), + wishlist=wishlist, + fallback_client=_DeezerStub(), + fallback_source='deezer', + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + + assert payload['enhanced_count'] == 1 + # Direct-lookup ran, text search did NOT (fast path skipped it). + assert ('details', '12345') in deezer_calls + assert not any(call[0] == 'search' for call in deezer_calls) + md = wishlist.added[0]['spotify_track_data'] + assert md['id'] == '12345' + assert md['artists'] == [{'name': 'Artist Name'}] + + +def test_direct_lookup_via_itunes_id_skips_text_search(): + """Stored itunes_track_id triggers iTunes direct lookup. Same + contract as Spotify / Deezer — get_track_details called, search + not.""" + itunes_calls = [] + enhanced_dict = { + 'id': 'it-9001', 'name': 'Track One', + 'artists': ['Artist Name'], + 'album': {'id': 'alb-it', 'name': 'Album X'}, + 'duration_ms': 180000, + } + + class _ItunesStub: + def get_track_details(self, tid): + itunes_calls.append(('details', tid)) + return enhanced_dict + def search_tracks(self, q, limit=10): + itunes_calls.append(('search', q)) + return [] + + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(itunes_tid='it-9001'), + wishlist=wishlist, + fallback_client=_ItunesStub(), + fallback_source='itunes', + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + + assert payload['enhanced_count'] == 1 + assert ('details', 'it-9001') in itunes_calls + assert not any(call[0] == 'search' for call in itunes_calls) + + +def test_direct_lookup_prefers_user_primary_source(): + """Track has stored IDs on multiple sources; user's configured + primary source is tried first. Pin: a Deezer-primary user with + both spotify_track_id and deezer_id stored gets the Deezer payload + (correct cover art / album shape for their setup), not whichever + source happened to come first in registry order. + """ + spotify_calls = [] + deezer_calls = [] + + spotify_enhanced = { + 'id': 'sp-1', 'name': 'Track One', + 'artists': ['Artist Name'], + 'album': {'id': 'sp-alb', 'name': 'Album X'}, + } + deezer_enhanced = { + 'id': 'dz-1', 'name': 'Track One', + 'artists': ['Artist Name'], + 'album': {'id': 'dz-alb', 'name': 'Album X'}, + } + + class _SpotifyStub: + def get_track_details(self, tid): + spotify_calls.append(tid) + return spotify_enhanced + + class _DeezerStub: + def get_track_details(self, tid): + deezer_calls.append(tid) + return deezer_enhanced + + # Patch get_primary_source to return 'deezer' so we don't depend + # on the test-runner's actual config. + import core.metadata.registry as registry_mod + original = registry_mod.get_primary_source + registry_mod.get_primary_source = lambda **kw: 'deezer' + try: + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=_SpotifyStub(), + artist_detail=_artist_with_track( + spotify_tid='sp-stored', deezer_tid='dz-stored', + ), + wishlist=wishlist, + fallback_client=_DeezerStub(), + fallback_source='deezer', + ) + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + finally: + registry_mod.get_primary_source = original + + assert payload['enhanced_count'] == 1 + # Deezer (primary) tried first and won — Spotify never queried. + assert deezer_calls == ['dz-stored'] + assert spotify_calls == [] + md = wishlist.added[0]['spotify_track_data'] + assert md['id'] == 'dz-1' + + +def test_direct_lookup_falls_through_to_text_search_when_no_stored_ids(): + """Track has ZERO stored source IDs → direct lookup yields nothing + → falls through to multi-source text search. Pin the contract: + direct-lookup is best-effort, not required.""" + text_search_track = _SpotifyTrack(id='dz-search', name='Track One', + artists=['Artist Name']) + text_search_track.album = 'Album X' + + class _DeezerStub: + def get_track_details(self, tid): + raise AssertionError("should not be called — no stored ID") + def search_tracks(self, q, limit=10): + return [text_search_track] + + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(), # no stored IDs + wishlist=wishlist, + fallback_client=_DeezerStub(), + fallback_source='deezer', + ) + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + assert payload['enhanced_count'] == 1 + assert wishlist.added[0]['spotify_track_data']['id'] == 'dz-search' + + +def test_direct_lookup_failure_falls_through_to_text_search(): + """Stored ID exists but direct lookup returns None (network blip, + catalog removal, etc.) → flow falls through to text search rather + than hard-failing. Pin: direct-lookup miss is non-fatal.""" + text_search_track = _SpotifyTrack(id='dz-search', name='Track One', + artists=['Artist Name']) + text_search_track.album = 'Album X' + + class _DeezerStub: + def get_track_details(self, tid): + return None # direct lookup miss + def search_tracks(self, q, limit=10): + return [text_search_track] + + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(deezer_tid='9999'), + wishlist=wishlist, + fallback_client=_DeezerStub(), + fallback_source='deezer', + ) + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + assert payload['enhanced_count'] == 1 + # Text-search winner used. + assert wishlist.added[0]['spotify_track_data']['id'] == 'dz-search' + + # --------------------------------------------------------------------------- # Failure modes # --------------------------------------------------------------------------- @@ -266,21 +600,139 @@ def test_track_with_no_file_path_marked_failed(): def test_no_match_anywhere_marked_failed(): - """No Spotify match AND no fallback match → failed reason 'No Spotify or fallback match'.""" + """No source returns a usable match → failed reason lists the + sources that were searched so user knows what was tried.""" spotify = _FakeSpotify(track_details=None, search_results=[]) - fallback = type('FB', (), { - 'search_tracks': lambda self, q, limit=5: [], - })() deps = _build_deps( spotify=spotify, artist_detail=_artist_with_track(), - fallback_client=fallback, ) payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) assert payload['failed_count'] == 1 - assert 'No Spotify or fallback match' in payload['failed_tracks'][0]['reason'] + reason = payload['failed_tracks'][0]['reason'] + assert 'No usable match across' in reason + assert 'spotify' in reason + assert 'try connecting an additional metadata source' in reason + + +def test_no_match_without_spotify_lists_searched_sources(): + """When Spotify isn't connected and the configured sources find + nothing, the failure reason lists every searched source. Discord + case: user with no Spotify / Deezer saw enhance silently produce + 'unknown artist - unknown album - unknown track' wishlist entries + instead of a clear failure.""" + fallback = type('FB', (), { + 'search_tracks': lambda self, q, limit=5: [], + })() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(), + fallback_client=fallback, + fallback_source='itunes', + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + assert payload['failed_count'] == 1 + reason = payload['failed_tracks'][0]['reason'] + assert 'No usable match across' in reason + assert 'itunes' in reason + + +def test_no_match_with_no_sources_configured_prompts_setup(): + """User with literally zero metadata sources configured gets a + clear prompt to connect one — instead of a generic 'no match'.""" + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(), + search_sources=[], + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + assert payload['failed_count'] == 1 + reason = payload['failed_tracks'][0]['reason'] + assert 'No metadata source configured' in reason + + +def test_fallback_match_with_empty_artist_rejected(): + """Per the user-reported bug: an iTunes match that clears the 0.7 + confidence threshold but has empty/missing artists is rejected + instead of producing a wishlist entry with empty artist field + (which the wishlist payload normalizer happily accepts and the + UI then displays as 'unknown artist').""" + fallback_track = _SpotifyTrack(id='it-empty', name='Track One', + artists=[], # empty artists list + image_url='') + fallback_track.track_number = 1 + fallback_track.disc_number = 1 + fallback_track.album = 'Album X' + fallback = type('FB', (), { + 'search_tracks': lambda self, q, limit=5: [fallback_track], + })() + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(), + wishlist=wishlist, + fallback_client=fallback, + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + + # No wishlist entry with empty fields — match was rejected. + assert payload['enhanced_count'] == 0 + assert payload['failed_count'] == 1 + assert wishlist.added == [] + + +def test_fallback_match_with_empty_album_rejected(): + """Empty album field on iTunes match → reject (was producing + 'unknown album' wishlist entries).""" + fallback_track = _SpotifyTrack(id='it-no-album', name='Track One', + artists=['Artist Name']) + fallback_track.track_number = 1 + fallback_track.disc_number = 1 + fallback_track.album = '' # empty + fallback = type('FB', (), { + 'search_tracks': lambda self, q, limit=5: [fallback_track], + })() + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(), + wishlist=wishlist, + fallback_client=fallback, + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + assert payload['enhanced_count'] == 0 + assert payload['failed_count'] == 1 + assert wishlist.added == [] + + +def test_fallback_match_with_empty_name_rejected(): + """Empty title on iTunes match → reject.""" + fallback_track = _SpotifyTrack(id='it-no-name', name='', + artists=['Artist Name']) + fallback_track.track_number = 1 + fallback_track.disc_number = 1 + fallback_track.album = 'Album X' + fallback = type('FB', (), { + 'search_tracks': lambda self, q, limit=5: [fallback_track], + })() + wishlist = _FakeWishlist() + deps = _build_deps( + spotify=None, + artist_detail=_artist_with_track(), + wishlist=wishlist, + fallback_client=fallback, + ) + + payload, _ = aq.enhance_artist_quality('artist-1', ['t1'], deps) + assert payload['enhanced_count'] == 0 + assert payload['failed_count'] == 1 + assert wishlist.added == [] # --------------------------------------------------------------------------- diff --git a/tests/conftest.py b/tests/conftest.py index 390252ea..2ee99421 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,8 @@ from flask_socketio import SocketIO, join_room, leave_room # --------------------------------------------------------------------------- _DEFAULT_STATUS_CACHE = { - 'spotify': {'connected': True, 'authenticated': True, 'response_time': 12.5, 'source': 'spotify'}, + 'metadata_source': {'connected': True, 'response_time': 12.5, 'source': 'spotify'}, + 'spotify': {'connected': True, 'authenticated': True, 'rate_limited': False, 'rate_limit': None, 'post_ban_cooldown': None}, 'media_server': {'connected': True, 'response_time': 8.1, 'type': 'plex'}, 'soulseek': {'connected': True, 'response_time': 5.3, 'source': 'soulseek'}, } @@ -255,6 +256,7 @@ wishlist_stats_state = copy.deepcopy(_DEFAULT_WISHLIST_STATS) def _build_status_payload(): return { + 'metadata_source': dict(_status_cache['metadata_source']), 'spotify': dict(_status_cache['spotify']), 'media_server': dict(_status_cache['media_server']), 'soulseek': dict(_status_cache['soulseek']), @@ -313,11 +315,11 @@ ENRICHMENT_WORKERS = [ ] ENRICHMENT_ENDPOINTS = { - 'musicbrainz': '/api/musicbrainz/status', - 'audiodb': '/api/audiodb/status', - 'deezer': '/api/deezer/status', - 'spotify-enrichment': '/api/spotify-enrichment/status', - 'itunes-enrichment': '/api/itunes-enrichment/status', + 'musicbrainz': '/api/enrichment/musicbrainz/status', + 'audiodb': '/api/enrichment/audiodb/status', + 'deezer': '/api/enrichment/deezer/status', + 'spotify-enrichment': '/api/enrichment/spotify/status', + 'itunes-enrichment': '/api/enrichment/itunes/status', 'hydrabase': '/api/hydrabase-worker/status', 'repair': '/api/repair/status', } @@ -576,23 +578,23 @@ def test_app(): # --- Phase 3 HTTP endpoints (enrichment workers) --- - @app.route('/api/musicbrainz/status') + @app.route('/api/enrichment/musicbrainz/status') def musicbrainz_status(): return jsonify(_build_enrichment_status('musicbrainz')) - @app.route('/api/audiodb/status') + @app.route('/api/enrichment/audiodb/status') def audiodb_status(): return jsonify(_build_enrichment_status('audiodb')) - @app.route('/api/deezer/status') + @app.route('/api/enrichment/deezer/status') def deezer_status(): return jsonify(_build_enrichment_status('deezer')) - @app.route('/api/spotify-enrichment/status') + @app.route('/api/enrichment/spotify/status') def spotify_enrichment_status(): return jsonify(_build_enrichment_status('spotify-enrichment')) - @app.route('/api/itunes-enrichment/status') + @app.route('/api/enrichment/itunes/status') def itunes_enrichment_status(): return jsonify(_build_enrichment_status('itunes-enrichment')) diff --git a/tests/discovery/test_discovery_sync.py b/tests/discovery/test_discovery_sync.py index f8fa3119..8cd9eee0 100644 --- a/tests/discovery/test_discovery_sync.py +++ b/tests/discovery/test_discovery_sync.py @@ -35,6 +35,15 @@ class _FakeMediaClient: return self._connected +class _FakeMediaServerEngine: + """Stand-in for MediaServerEngine — only the bits SyncDeps needs.""" + def __init__(self, plex=None, jellyfin=None, navidrome=None): + self._clients = {'plex': plex, 'jellyfin': jellyfin, 'navidrome': navidrome} + + def client(self, name): + return self._clients.get(name) + + class _FakeSyncService: def __init__(self, *, media_client=None, server_type='plex', sync_result=None, raise_on_sync=None, @@ -44,8 +53,12 @@ class _FakeSyncService: self._sync_result = sync_result or _FakeSyncResult() self._raise_on_sync = raise_on_sync self.spotify_client = object() if spotify_client else None - self.plex_client = object() if plex_client else None - self.jellyfin_client = object() if jellyfin_client else None + # The sync_service exposes the engine so the discovery worker + # can introspect per-server clients via self._engine.client(name). + self._engine = _FakeMediaServerEngine( + plex=object() if plex_client else None, + jellyfin=object() if jellyfin_client else None, + ) self.progress_callback = None self.progress_playlist_name = None self.cleared_callbacks = [] @@ -130,8 +143,10 @@ def _build_deps( return ds.SyncDeps( config_manager=config or _FakeConfig(), sync_service=sync_service or _FakeSyncService(media_client=_FakeMediaClient()), - plex_client=plex or _FakePlex(), - jellyfin_client=jellyfin or _FakeJellyfin(), + media_server_engine=_FakeMediaServerEngine( + plex=plex or _FakePlex(), + jellyfin=jellyfin or _FakeJellyfin(), + ), automation_engine=automation or _FakeAutomationEngine(), run_async=run_async or _run_async_sync, record_sync_history_start=record_sync_history_start or (lambda **kw: None), diff --git a/tests/discovery/test_quality_scanner_typed_album.py b/tests/discovery/test_quality_scanner_typed_album.py new file mode 100644 index 00000000..16cebacc --- /dev/null +++ b/tests/discovery/test_quality_scanner_typed_album.py @@ -0,0 +1,87 @@ +"""Pin the typed-path migration of `_normalize_track_album` in +`core/discovery/quality_scanner.py`. + +Quality scanner result normalization now routes the embedded +`track.album` blob through `Album.from__dict()` when provider +is known. Falls back to legacy duck-typed extraction below. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from core.discovery import quality_scanner + + +SPOTIFY_TRACK = { + 'id': 'tr1', + 'name': 'HUMBLE.', + 'artists': [{'id': 'kdot', 'name': 'Kendrick Lamar'}], + 'album': { + 'id': 'sp_album', + 'name': 'DAMN.', + 'artists': [{'id': 'kdot', 'name': 'Kendrick Lamar'}], + 'release_date': '2017-04-14', + 'total_tracks': 14, + 'album_type': 'album', + 'images': [{'url': 'https://i.scdn.co/640.jpg'}], + }, +} + + +def test_typed_path_seeds_album_fields_from_known_provider(): + out = quality_scanner._normalize_track_album(SPOTIFY_TRACK, provider='spotify') + assert out['name'] == 'DAMN.' + assert out['album_type'] == 'album' + assert out['total_tracks'] == 14 + assert out['release_date'] == '2017-04-14' + assert out['id'] == 'sp_album' + + +def test_legacy_path_used_when_no_provider(): + out = quality_scanner._normalize_track_album(SPOTIFY_TRACK) + # Legacy path still resolves album from raw `album` dict. + assert out['name'] == 'DAMN.' + assert out['album_type'] == 'album' + assert out['total_tracks'] == 14 + + +def test_legacy_path_used_for_unknown_provider(): + out = quality_scanner._normalize_track_album(SPOTIFY_TRACK, provider='made_up') + assert out['name'] == 'DAMN.' + + +def test_legacy_path_used_when_typed_converter_raises(): + def _explode(_): + raise RuntimeError('simulated converter bug') + + with patch.dict(quality_scanner._TYPED_ALBUM_CONVERTERS, + {'spotify': _explode}): + out = quality_scanner._normalize_track_album(SPOTIFY_TRACK, provider='spotify') + # Fell back to legacy — still has the album fields. + assert out['name'] == 'DAMN.' + assert out['total_tracks'] == 14 + + +@pytest.mark.parametrize('provider,track', [ + ('itunes', { + 'id': 1, 'name': 'X', + 'album': {'collectionId': 99, 'collectionName': 'iTunes Album', + 'artistName': 'Artist', 'trackCount': 10}, + }), + ('deezer', { + 'id': 2, 'name': 'X', + 'album': {'id': 99, 'title': 'Deezer Album', + 'artist': {'name': 'Artist'}, 'nb_tracks': 8}, + }), + ('discogs', { + 'id': 3, 'name': 'X', + 'album': {'id': 99, 'title': 'Discogs Album', + 'artists': [{'name': 'Artist'}], 'year': 2020}, + }), +]) +def test_typed_path_works_for_other_providers(provider, track): + out = quality_scanner._normalize_track_album(track, provider=provider) + assert out['name'] # populated diff --git a/tests/downloads/test_background_download_worker.py b/tests/downloads/test_background_download_worker.py new file mode 100644 index 00000000..d96d49f8 --- /dev/null +++ b/tests/downloads/test_background_download_worker.py @@ -0,0 +1,542 @@ +"""Tests for `BackgroundDownloadWorker` (Phase C1). + +These tests pin the worker's state-machine semantics, semaphore +serialization, rate-limit-delay behavior, and exception handling. +Future phases (C2–C7) migrate each per-source client onto this +worker — these tests stay green as the regression net. +""" + +from __future__ import annotations + +import threading +import time + +from core.download_engine import DownloadEngine + + +# --------------------------------------------------------------------------- +# Dispatch — initial state + thread spawn +# --------------------------------------------------------------------------- + + +def test_dispatch_returns_uuid_download_id(): + engine = DownloadEngine() + + def impl(download_id, target_id, display_name): + return '/tmp/file.flac' + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='abc123', + display_name='Some Song', + original_filename='abc123||Some Song', + impl_callable=impl, + ) + assert len(download_id) == 36 # UUID4 + assert download_id.count('-') == 4 + + +def test_dispatch_inserts_initial_record_with_canonical_state(): + """Pinning: initial record matches the legacy per-client shape so + consumers reading the state dict via API or context-key lookup + keep working unchanged after migration.""" + engine = DownloadEngine() + captured = threading.Event() + + def impl(download_id, target_id, display_name): + captured.wait(timeout=1.0) # block so we can read 'Initializing' / 'InProgress' state + return '/tmp/file.flac' + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='abc', + display_name='X', + original_filename='abc||X', + impl_callable=impl, + ) + record = engine.get_record('youtube', download_id) + assert record is not None + assert record['id'] == download_id + assert record['filename'] == 'abc||X' + assert record['username'] == 'youtube' + assert record['state'] in ('Initializing', 'InProgress, Downloading') + assert record['progress'] == 0.0 + assert record['file_path'] is None + captured.set() # release impl + + +def test_dispatch_merges_extra_record_fields(): + """Pinning: source-specific slots (video_id, track_id, etc.) + merge into the initial record so frontend + status APIs that + read those keys keep working.""" + engine = DownloadEngine() + started = threading.Event() + release = threading.Event() + + def impl(download_id, target_id, display_name): + started.set() + release.wait(timeout=1.0) + return '/tmp/x.flac' + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='vid123', + display_name='Title', + original_filename='vid123||Title', + impl_callable=impl, + extra_record_fields={ + 'video_id': 'vid123', + 'url': 'https://youtube.com/watch?v=vid123', + 'title': 'Title', + }, + ) + started.wait(timeout=1.0) + record = engine.get_record('youtube', download_id) + assert record['video_id'] == 'vid123' + assert record['url'] == 'https://youtube.com/watch?v=vid123' + assert record['title'] == 'Title' + release.set() + + +def test_dispatch_username_override_preserves_legacy_slot(): + """Pinning: Deezer's record stores `'deezer_dl'` (legacy) in the + username slot, not the canonical `'deezer'`. Worker accepts + override so frontend status indicators keep their key.""" + engine = DownloadEngine() + release = threading.Event() + + def impl(download_id, target_id, display_name): + release.wait(timeout=1.0) + return '/tmp/x.flac' + + download_id = engine.worker.dispatch( + source_name='deezer', + target_id='999', + display_name='X', + original_filename='999||X', + impl_callable=impl, + username_override='deezer_dl', + ) + record = engine.get_record('deezer', download_id) + assert record['username'] == 'deezer_dl' + release.set() + + +# --------------------------------------------------------------------------- +# Worker lifecycle — state transitions +# --------------------------------------------------------------------------- + + +def test_worker_marks_completed_on_successful_impl(): + engine = DownloadEngine() + + def impl(download_id, target_id, display_name): + return '/tmp/done.flac' + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='vid', + display_name='X', + original_filename='vid||X', + impl_callable=impl, + ) + + # Wait for thread to finish. + deadline = time.time() + 2.0 + while time.time() < deadline: + record = engine.get_record('youtube', download_id) + if record and record['state'] == 'Completed, Succeeded': + break + time.sleep(0.01) + + record = engine.get_record('youtube', download_id) + assert record['state'] == 'Completed, Succeeded' + assert record['progress'] == 100.0 + assert record['file_path'] == '/tmp/done.flac' + + +def test_worker_preserves_cancelled_when_impl_returns_none(): + """Pinning: if the user cancels mid-download (state flips to + Cancelled via engine.update_record from cancel_download), the + worker must NOT clobber it back to Errored when impl returns + None. The legacy per-client thread workers had this guard + (``if state != 'Cancelled': state = 'Errored'``); the shared + worker preserves that contract.""" + engine = DownloadEngine() + + def impl(download_id, target_id, display_name): + # Simulate user cancelling mid-impl by writing Cancelled. + engine.update_record('youtube', download_id, {'state': 'Cancelled'}) + return None # impl returns None because download was interrupted + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='vid', + display_name='X', + original_filename='vid||X', + impl_callable=impl, + ) + + deadline = time.time() + 2.0 + while time.time() < deadline: + record = engine.get_record('youtube', download_id) + if record and record['state'] in ('Cancelled', 'Errored'): + break + time.sleep(0.01) + + record = engine.get_record('youtube', download_id) + assert record['state'] == 'Cancelled', ( + f"Worker clobbered user's Cancelled with {record['state']}" + ) + + +def test_worker_preserves_cancelled_when_impl_returns_success(): + """Cin's bug 3 follow-up: the success path also has a read-then-write + race. If the user cancels between the impl returning a valid file + path and the worker writing 'Completed, Succeeded', the cancel is + overwritten. The success-path write must use the same atomic + Cancelled-preserve guard as _mark_terminal.""" + engine = DownloadEngine() + + def impl(download_id, target_id, display_name): + # User cancels mid-impl, then impl finishes successfully. + engine.update_record('youtube', download_id, {'state': 'Cancelled'}) + return '/tmp/file.flac' + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='vid', + display_name='X', + original_filename='vid||X', + impl_callable=impl, + ) + + deadline = time.time() + 2.0 + while time.time() < deadline: + record = engine.get_record('youtube', download_id) + if record and record['state'] in ('Cancelled', 'Completed, Succeeded'): + break + time.sleep(0.01) + + record = engine.get_record('youtube', download_id) + assert record['state'] == 'Cancelled', ( + f"Worker clobbered user's Cancelled with {record['state']}" + ) + + +def test_worker_preserves_cancelled_when_impl_raises(): + """Same Cancelled-preserve guard, but for the impl-raises path.""" + engine = DownloadEngine() + + def impl(download_id, target_id, display_name): + engine.update_record('youtube', download_id, {'state': 'Cancelled'}) + raise RuntimeError("simulated mid-cancel exception") + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='vid', + display_name='X', + original_filename='vid||X', + impl_callable=impl, + ) + + deadline = time.time() + 2.0 + while time.time() < deadline: + record = engine.get_record('youtube', download_id) + if record and record['state'] in ('Cancelled', 'Errored'): + break + time.sleep(0.01) + + assert engine.get_record('youtube', download_id)['state'] == 'Cancelled' + + +def test_worker_marks_errored_when_impl_returns_none(): + engine = DownloadEngine() + + def impl(download_id, target_id, display_name): + return None # signaling failure + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='vid', + display_name='X', + original_filename='vid||X', + impl_callable=impl, + ) + + deadline = time.time() + 2.0 + while time.time() < deadline: + record = engine.get_record('youtube', download_id) + if record and record['state'] == 'Errored': + break + time.sleep(0.01) + + record = engine.get_record('youtube', download_id) + assert record['state'] == 'Errored' + # file_path stays None (default). + assert record['file_path'] is None + + +def test_worker_marks_errored_and_captures_message_when_impl_raises(): + engine = DownloadEngine() + + def impl(download_id, target_id, display_name): + raise RuntimeError("api blew up") + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='vid', + display_name='X', + original_filename='vid||X', + impl_callable=impl, + ) + + deadline = time.time() + 2.0 + while time.time() < deadline: + record = engine.get_record('youtube', download_id) + if record and record['state'] == 'Errored': + break + time.sleep(0.01) + + record = engine.get_record('youtube', download_id) + assert record['state'] == 'Errored' + assert 'api blew up' in record.get('error', '') + + +# --------------------------------------------------------------------------- +# Per-source semaphore serialization +# --------------------------------------------------------------------------- + + +def test_semaphore_serializes_downloads_for_same_source(): + """Pinning: with concurrency=1 (default), two dispatches against + the same source run sequentially. The legacy per-client + semaphore did the same — consumers depend on this for + rate-limit safety against APIs like YouTube.""" + engine = DownloadEngine() + in_progress = threading.Event() + can_finish = threading.Event() + overlap_count = 0 + overlap_lock = threading.Lock() + active_count = [0] + + def impl(download_id, target_id, display_name): + nonlocal overlap_count + with overlap_lock: + active_count[0] += 1 + if active_count[0] > 1: + overlap_count += 1 + in_progress.set() + can_finish.wait(timeout=2.0) + with overlap_lock: + active_count[0] -= 1 + return '/tmp/x.flac' + + # Default concurrency=1 — two dispatches must serialize. + dl1 = engine.worker.dispatch( + source_name='youtube', target_id='a', display_name='A', + original_filename='a||A', impl_callable=impl, + ) + in_progress.wait(timeout=1.0) + in_progress.clear() + dl2 = engine.worker.dispatch( + source_name='youtube', target_id='b', display_name='B', + original_filename='b||B', impl_callable=impl, + ) + # Give second dispatch a chance to attempt running in parallel + # (it should be blocked on the semaphore). + time.sleep(0.1) + assert overlap_count == 0, "second dispatch should be blocked behind semaphore" + + # Release first; second proceeds. + can_finish.set() + + # Wait for both to finish. + deadline = time.time() + 3.0 + while time.time() < deadline: + r1 = engine.get_record('youtube', dl1) + r2 = engine.get_record('youtube', dl2) + if r1 and r2 and r1['state'] == 'Completed, Succeeded' and r2['state'] == 'Completed, Succeeded': + break + time.sleep(0.01) + + assert overlap_count == 0 + + +def test_semaphore_concurrency_can_be_increased(): + """When `set_concurrency(source, N)` is called, N downloads can + run in parallel for that source. Used by sources that support + parallel transfers (none today, but contract supports it).""" + engine = DownloadEngine() + engine.worker.set_concurrency('parallel-source', 3) + + in_flight = [] + in_flight_lock = threading.Lock() + can_finish = threading.Event() + max_observed = [0] + + def impl(download_id, target_id, display_name): + with in_flight_lock: + in_flight.append(download_id) + max_observed[0] = max(max_observed[0], len(in_flight)) + can_finish.wait(timeout=2.0) + with in_flight_lock: + in_flight.remove(download_id) + return '/tmp/x.flac' + + for i in range(3): + engine.worker.dispatch( + source_name='parallel-source', + target_id=str(i), + display_name=f'd{i}', + original_filename=f'{i}||d{i}', + impl_callable=impl, + ) + # Give threads time to ramp up. + time.sleep(0.2) + can_finish.set() + + # Wait for them to finish. + time.sleep(0.5) + assert max_observed[0] == 3 + + +# --------------------------------------------------------------------------- +# Per-source rate-limit delay +# --------------------------------------------------------------------------- + + +def test_impl_can_observe_cancel_mid_flight_via_state_check(): + """Per JohnBaumb: existing tests cover Cancelled-preserve AFTER impl + returns — but plugins also poll engine state mid-download (via + ``_is_cancelled`` helpers) to abort partial transfers. Pin that + contract: a cancel landing while impl is mid-flight must be + visible to a subsequent ``engine.get_record()`` from the impl + thread. + """ + engine = DownloadEngine() + impl_started = threading.Event() + impl_can_finish = threading.Event() + observed_state_during_impl = [] + + def impl(download_id, target_id, display_name): + impl_started.set() + # Wait for the test thread to write Cancelled, then check + # what we can observe from inside the impl callback. + impl_can_finish.wait(timeout=2.0) + record = engine.get_record('youtube', download_id) + observed_state_during_impl.append(record.get('state') if record else None) + return None # impl noticed cancel + bailed out + + download_id = engine.worker.dispatch( + source_name='youtube', + target_id='vid', + display_name='X', + original_filename='vid||X', + impl_callable=impl, + ) + + # Wait for impl to start, then inject a cancel. + impl_started.wait(timeout=1.0) + engine.update_record('youtube', download_id, {'state': 'Cancelled'}) + impl_can_finish.set() + + # Wait for impl to finish (its append populates observed_state). + # Engine state is already Cancelled at this point, so polling on + # state would race: it'd break before impl ran the get_record line. + deadline = time.time() + 2.0 + while not observed_state_during_impl and time.time() < deadline: + time.sleep(0.01) + + # impl observed the Cancelled state mid-flight via get_record, + # AND the worker preserved Cancelled after impl returned None. + assert observed_state_during_impl == ['Cancelled'] + assert engine.get_record('youtube', download_id)['state'] == 'Cancelled' + + +def test_per_source_delays_dont_block_other_sources(): + """Per JohnBaumb: per-source semaphores + delays must not let one + slow source stall another. YouTube's 3s rate-limit delay should + not delay a Tidal download starting in parallel. + + Configure YouTube with a 0.5s delay, dispatch one YouTube download + (which holds the source's serial slot + arms the next-call delay), + then immediately dispatch a Tidal download. Tidal must complete + well before YouTube's delay window would have elapsed. + """ + engine = DownloadEngine() + engine.worker.set_delay('youtube', 0.5) + + yt_completed = threading.Event() + td_completed = threading.Event() + + def yt_impl(download_id, target_id, display_name): + time.sleep(0.05) + yt_completed.set() + return '/tmp/yt.mp3' + + def td_impl(download_id, target_id, display_name): + td_completed.set() + return '/tmp/td.flac' + + # First YouTube call. After it finishes, the worker arms the + # 0.5s delay BEFORE the next youtube dispatch can run. + engine.worker.dispatch( + source_name='youtube', target_id='a', display_name='A', + original_filename='a||A', impl_callable=yt_impl, + ) + yt_completed.wait(timeout=1.0) + + # Second YouTube would now block 0.5s on the rate-limit delay. + # Dispatch one to occupy that wait, then dispatch Tidal — Tidal + # must NOT wait on YouTube's delay. + engine.worker.dispatch( + source_name='youtube', target_id='b', display_name='B', + original_filename='b||B', impl_callable=lambda *a: '/tmp/y.mp3', + ) + + td_start = time.time() + engine.worker.dispatch( + source_name='tidal', target_id='t1', display_name='T', + original_filename='t1||T', impl_callable=td_impl, + ) + td_completed.wait(timeout=0.4) + td_elapsed = time.time() - td_start + + # Tidal must have finished in well under 0.5s (the YouTube delay). + assert td_completed.is_set(), "Tidal blocked on YouTube's per-source delay" + assert td_elapsed < 0.4, f"Tidal took {td_elapsed:.2f}s — should be near-instant" + + +def test_delay_enforces_minimum_gap_between_downloads(): + """Pinning: YouTube uses 3s delay today (legacy + `_download_delay`). Worker-driven delay must enforce the same + gap so YouTube doesn't 429.""" + engine = DownloadEngine() + engine.worker.set_delay('youtube', 0.2) # 200ms — short for test speed + + completion_times = [] + + def impl(download_id, target_id, display_name): + completion_times.append(time.time()) + return '/tmp/x.flac' + + # Two back-to-back dispatches. + engine.worker.dispatch( + source_name='youtube', target_id='a', display_name='A', + original_filename='a||A', impl_callable=impl, + ) + engine.worker.dispatch( + source_name='youtube', target_id='b', display_name='B', + original_filename='b||B', impl_callable=impl, + ) + + # Wait for both to finish (semaphore serializes + delay). + deadline = time.time() + 3.0 + while time.time() < deadline and len(completion_times) < 2: + time.sleep(0.01) + + assert len(completion_times) == 2 + gap = completion_times[1] - completion_times[0] + # Gap is at LEAST the configured delay. + assert gap >= 0.18, f"expected gap >= 0.2s, got {gap:.3f}" diff --git a/tests/downloads/test_deezer_pinning.py b/tests/downloads/test_deezer_pinning.py new file mode 100644 index 00000000..f3f2cab2 --- /dev/null +++ b/tests/downloads/test_deezer_pinning.py @@ -0,0 +1,154 @@ +"""Phase A pinning tests for DeezerDownloadClient — UPDATED for Phase C6. + +Deezer has the same engine-driven dispatch as the other streaming +sources, with three Deezer-specific quirks preserved: +- track_id stays as STRING (Deezer GW API uses string IDs). +- Engine record's `username` slot is the legacy `'deezer_dl'` + via worker username_override. +- Worker thread is named `deezer-dl-` for diagnostics. +""" + +from __future__ import annotations + +import asyncio +import threading +from pathlib import Path +from unittest.mock import patch + +import pytest + +from core.download_engine import DownloadEngine +from core.deezer_download_client import DeezerDownloadClient + + +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture +def deezer_client_with_engine(): + client = DeezerDownloadClient.__new__(DeezerDownloadClient) + client.download_path = Path('./test_deezer_downloads') + client.shutdown_check = None + client._authenticated = True + client._engine = None + engine = DownloadEngine() + client.set_engine(engine) + return client, engine + + +def test_download_returns_none_when_not_authenticated(deezer_client_with_engine): + client, _ = deezer_client_with_engine + client._authenticated = False + result = _run_async(client.download('deezer_dl', '12345||x', 0)) + assert result is None + + +def test_download_raises_when_engine_not_wired(): + """Defensive: client without engine reference must raise so the + orchestrator's download_with_fallback surfaces the error and + moves on to the next source. Returning None silently would drop + the download with no user feedback (per JohnBaumb).""" + import pytest + client = DeezerDownloadClient.__new__(DeezerDownloadClient) + client._engine = None + # Bypass auth gate so we exercise the engine check. + client._authenticated = True + with pytest.raises(RuntimeError, match="engine reference"): + _run_async(client.download('deezer', 'v||t', 0)) + + +def test_download_track_id_stays_as_string(deezer_client_with_engine): + """Pinning: Deezer GW API uses string IDs — engine record must + keep track_id as str.""" + client, engine = deezer_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/done.flac' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download('deezer_dl', '999||X', 0)) + started.wait(timeout=1.0) + record = engine.get_record('deezer', download_id) + assert record['track_id'] == '999' + assert isinstance(record['track_id'], str) + release.set() + + +def test_download_username_slot_is_legacy_deezer_dl(deezer_client_with_engine): + """Pinning: frontend status indicators key off `'deezer_dl'`, + not the canonical `'deezer'`.""" + client, engine = deezer_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/done.flac' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download('deezer_dl', '999||x', 0)) + started.wait(timeout=1.0) + assert engine.get_record('deezer', download_id)['username'] == 'deezer_dl' + release.set() + + +def test_download_handles_missing_display_name_with_fallback(deezer_client_with_engine): + """Pinning: filename without `||` synthesizes display name `Track `.""" + client, engine = deezer_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/x.flac' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download('deezer_dl', '12345', 0)) + started.wait(timeout=1.0) + assert engine.get_record('deezer', download_id)['display_name'] == 'Track 12345' + release.set() + + +def test_download_engine_record_carries_error_slot(deezer_client_with_engine): + """Pinning: Deezer-specific `error` slot for ARL re-auth failure + messages must be present on init.""" + client, engine = deezer_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/x.flac' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download('deezer_dl', '999||X', 1024)) + started.wait(timeout=1.0) + record = engine.get_record('deezer', download_id) + assert 'error' in record + assert record['error'] is None + assert record['size'] == 1024 + release.set() + + +def test_get_all_downloads_reads_engine_records(deezer_client_with_engine): + client, engine = deezer_client_with_engine + engine.add_record('deezer', 'dl-1', { + 'id': 'dl-1', 'filename': '111||A', 'username': 'deezer_dl', + 'state': 'InProgress, Downloading', 'progress': 50.0, + }) + result = _run_async(client.get_all_downloads()) + assert len(result) == 1 + assert result[0].id == 'dl-1' + assert result[0].username == 'deezer_dl' diff --git a/tests/downloads/test_download_engine.py b/tests/downloads/test_download_engine.py new file mode 100644 index 00000000..16a680ea --- /dev/null +++ b/tests/downloads/test_download_engine.py @@ -0,0 +1,724 @@ +"""Tests for the DownloadEngine skeleton (Phase B). + +Pinning the engine's state-storage contract: add/update/remove, +per-source iteration, find-by-id, plugin registration, lock-held +mutations vs lock-released reads. Future phases (C/D/E/F) bolt +behavior on top of this surface — these tests stay green and act +as the regression net while behavior moves in. +""" + +from __future__ import annotations + +import threading + +import pytest + +from core.download_engine import DownloadEngine + + +# --------------------------------------------------------------------------- +# Plugin registration +# --------------------------------------------------------------------------- + + +def test_register_plugin_stores_under_source_name(): + engine = DownloadEngine() + plugin = object() + engine.register_plugin('soulseek', plugin) + assert engine.get_plugin('soulseek') is plugin + assert 'soulseek' in engine.registered_sources() + + +def test_get_plugin_returns_none_for_unknown_source(): + engine = DownloadEngine() + assert engine.get_plugin('made_up') is None + + +def test_register_plugin_swallows_set_engine_failure(caplog): + """Per JohnBaumb: if a plugin's ``set_engine`` callback raises, + registration shouldn't take down the engine — the failure gets + logged + the plugin stays registered. The plugin's own download() + method is responsible for surfacing the missing-engine state to + the user (see ``test_download_raises_when_engine_not_wired`` per + source). Pinning this so a future refactor can't accidentally + propagate the set_engine exception and crash boot. + """ + class _BrokenSetEngine: + def set_engine(self, engine): + raise RuntimeError("plugin's set_engine blew up") + + engine = DownloadEngine() + plugin = _BrokenSetEngine() + # Should not raise. + engine.register_plugin('flaky', plugin) + + # Plugin still registered + lookupable. + assert engine.get_plugin('flaky') is plugin + assert 'flaky' in engine.registered_sources() + + +def test_register_plugin_overwrites_on_duplicate(caplog): + """Re-registering under the same name overwrites and warns. Not a + common path but useful so test fixtures that build a fresh engine + can swap a mock plugin in without setup gymnastics.""" + engine = DownloadEngine() + first = object() + second = object() + engine.register_plugin('soulseek', first) + engine.register_plugin('soulseek', second) + assert engine.get_plugin('soulseek') is second + + +# --------------------------------------------------------------------------- +# Active-download state — add / get / update / remove +# --------------------------------------------------------------------------- + + +def test_add_record_inserts_under_composite_key(): + engine = DownloadEngine() + engine.add_record('youtube', 'dl-1', {'state': 'Initializing', 'progress': 0.0}) + + rec = engine.get_record('youtube', 'dl-1') + assert rec is not None + assert rec['state'] == 'Initializing' + assert rec['progress'] == 0.0 + + +def test_get_record_returns_shallow_copy(): + """Mutating the returned dict must NOT affect engine state. + Engine reads should be safe to hold / iterate without locks.""" + engine = DownloadEngine() + engine.add_record('youtube', 'dl-1', {'state': 'Initializing'}) + + rec = engine.get_record('youtube', 'dl-1') + rec['state'] = 'TamperedByCaller' + + # Engine state still has the original. + fresh = engine.get_record('youtube', 'dl-1') + assert fresh['state'] == 'Initializing' + + +def test_update_record_applies_partial_patch(): + engine = DownloadEngine() + engine.add_record('tidal', 'dl-2', {'state': 'Initializing', 'progress': 0.0, + 'file_path': None}) + + engine.update_record('tidal', 'dl-2', {'state': 'Completed, Succeeded', + 'progress': 100.0, + 'file_path': '/tmp/song.flac'}) + + rec = engine.get_record('tidal', 'dl-2') + assert rec['state'] == 'Completed, Succeeded' + assert rec['progress'] == 100.0 + assert rec['file_path'] == '/tmp/song.flac' + + +def test_update_record_is_noop_when_record_removed(): + """If a record was removed (e.g. user cancelled mid-download), + the worker thread's late update is silently dropped — never + raises. Mirrors the per-client `if download_id in active_downloads` + guard pattern that's all over the existing clients.""" + engine = DownloadEngine() + engine.add_record('tidal', 'dl-2', {'state': 'Initializing'}) + engine.remove_record('tidal', 'dl-2') + + # Should not raise. + engine.update_record('tidal', 'dl-2', {'state': 'Completed, Succeeded'}) + + assert engine.get_record('tidal', 'dl-2') is None + + +def test_remove_record_returns_removed_record(): + engine = DownloadEngine() + engine.add_record('qobuz', 'dl-3', {'state': 'InProgress'}) + + removed = engine.remove_record('qobuz', 'dl-3') + assert removed is not None + assert removed['state'] == 'InProgress' + assert engine.get_record('qobuz', 'dl-3') is None + + +def test_remove_record_returns_none_when_missing(): + engine = DownloadEngine() + assert engine.remove_record('qobuz', 'never-existed') is None + + +def test_per_source_locks_dont_block_each_other(): + """Per JohnBaumb: each source must have its own lock so a long-held + write on one source doesn't block writes on another. Pre-refactor + each client owned its own download lock; the engine has to match + that semantic. + + Hold source-A's lock from one thread, then mutate source-B from + another thread. Source-B's mutation must complete promptly even + while source-A is locked. + """ + engine = DownloadEngine() + engine.add_record('youtube', 'yt-1', {'state': 'InProgress'}) + engine.add_record('tidal', 'td-1', {'state': 'InProgress'}) + + held = threading.Event() + release = threading.Event() + other_done = threading.Event() + + def hold_youtube_lock(): + with engine._source_lock('youtube'): + held.set() + release.wait(timeout=2.0) + + def update_tidal_while_youtube_held(): + held.wait(timeout=1.0) + engine.update_record('tidal', 'td-1', {'state': 'Completed, Succeeded'}) + other_done.set() + + holder = threading.Thread(target=hold_youtube_lock) + other = threading.Thread(target=update_tidal_while_youtube_held) + holder.start() + other.start() + + # Tidal write must complete even though YouTube's lock is held. + assert other_done.wait(timeout=1.0), ( + "Tidal mutation blocked by YouTube's lock — sources are not " + "independently shardable" + ) + + release.set() + holder.join() + other.join() + assert engine.get_record('tidal', 'td-1')['state'] == 'Completed, Succeeded' + + +def test_remove_record_drops_empty_source_bucket(): + """Per JohnBaumb: nested layout makes per-source iteration + O(source_records). Removing the last record for a source must + also drop the empty source bucket so future iter_records_for_source + calls don't see a stale source key.""" + engine = DownloadEngine() + engine.add_record('youtube', 'yt-1', {'title': 'A'}) + engine.remove_record('youtube', 'yt-1') + + # Source bucket gone — internal state matches the contract. + assert 'youtube' not in engine._records + # Public surface still answers correctly. + assert list(engine.iter_records_for_source('youtube')) == [] + assert engine.get_record('youtube', 'yt-1') is None + + +# --------------------------------------------------------------------------- +# Iteration +# --------------------------------------------------------------------------- + + +def test_iter_records_for_source_filters_correctly(): + engine = DownloadEngine() + engine.add_record('youtube', 'yt-1', {'title': 'A'}) + engine.add_record('youtube', 'yt-2', {'title': 'B'}) + engine.add_record('tidal', 'td-1', {'title': 'C'}) + + yt_records = list(engine.iter_records_for_source('youtube')) + assert len(yt_records) == 2 + assert {r['title'] for r in yt_records} == {'A', 'B'} + + td_records = list(engine.iter_records_for_source('tidal')) + assert len(td_records) == 1 + assert td_records[0]['title'] == 'C' + + +def test_iter_yields_shallow_copies(): + """Iteration returns COPIES — caller can hold the list and mutate + each record without affecting engine state. Important: future + Phase B3's `get_all_downloads` will iterate then build + DownloadStatus objects from the snapshots.""" + engine = DownloadEngine() + engine.add_record('youtube', 'yt-1', {'title': 'A'}) + + snapshot = list(engine.iter_records_for_source('youtube')) + snapshot[0]['title'] = 'TAMPERED' + + fresh = engine.get_record('youtube', 'yt-1') + assert fresh['title'] == 'A' + + +# --------------------------------------------------------------------------- +# Thread safety — basic concurrent-mutation smoke +# --------------------------------------------------------------------------- + + +def test_concurrent_adds_dont_lose_records(): + """Hammer the engine with concurrent add_record from multiple + threads. With proper locking, every record lands in state. + Future Phase C BackgroundDownloadWorker spawns N threads doing + exactly this kind of mutation.""" + engine = DownloadEngine() + + def add_records(source, base): + for i in range(50): + engine.add_record(source, f'{base}-{i}', {'i': i}) + + threads = [ + threading.Thread(target=add_records, args=(f'src-{n}', f'dl-{n}')) + for n in range(4) + ] + for t in threads: + t.start() + for t in threads: + t.join() + + total = sum( + 1 + for n in range(4) + for _ in engine.iter_records_for_source(f'src-{n}') + ) + assert total == 4 * 50 # 200 records, none lost + + +# --------------------------------------------------------------------------- +# Cross-source query dispatch (Phase B2) +# --------------------------------------------------------------------------- + + +def _run_async(coro): + import asyncio + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +class _FakePlugin: + """Minimal plugin double for engine query tests. Exposes the + methods engine.get_all_downloads / get_download_status / + cancel_download / clear_all_completed_downloads call.""" + + def __init__(self, name, configured=True, downloads=None, + cancel_result=True, clear_result=True): + self.name = name + self._configured = configured + self._downloads = downloads or [] + self._cancel_result = cancel_result + self._clear_result = clear_result + self.cancel_calls = [] + self.clear_calls = 0 + + def is_configured(self): + return self._configured + + async def get_all_downloads(self): + return list(self._downloads) + + async def get_download_status(self, download_id): + for d in self._downloads: + if getattr(d, 'id', None) == download_id: + return d + return None + + async def cancel_download(self, download_id, source_hint, remove): + self.cancel_calls.append((download_id, source_hint, remove)) + return self._cancel_result + + async def clear_all_completed_downloads(self): + self.clear_calls += 1 + return self._clear_result + + +class _FakeStatus: + def __init__(self, id, source): + self.id = id + self.source = source + + +def test_engine_get_all_downloads_aggregates_across_plugins(): + """Engine concatenates every plugin's get_all_downloads output — + same behavior as the legacy orchestrator.""" + engine = DownloadEngine() + yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')]) + td_plugin = _FakePlugin('tidal', downloads=[_FakeStatus('td-1', 'tidal'), + _FakeStatus('td-2', 'tidal')]) + engine.register_plugin('youtube', yt_plugin) + engine.register_plugin('tidal', td_plugin) + + result = _run_async(engine.get_all_downloads()) + assert len(result) == 3 + assert {r.id for r in result} == {'yt-1', 'td-1', 'td-2'} + + +def test_engine_get_all_downloads_excludes_dont_invoke_plugin(): + """Per JohnBaumb: the monitor calls engine.get_all_downloads( + exclude=('soulseek',)) AFTER already pulling slskd transfers via + the transfers/downloads endpoint. The whole point of the exclude + is to NOT touch soulseek's plugin a second time — so the plugin's + get_all_downloads must literally not be called when its name is + in the exclude list. Pin that semantic explicitly (a test that + just checks IDs would pass even if soulseek was called and + returned []).""" + engine = DownloadEngine() + sl_plugin = _FakePlugin('soulseek', downloads=[_FakeStatus('sl-1', 'soulseek')]) + yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')]) + engine.register_plugin('soulseek', sl_plugin) + engine.register_plugin('youtube', yt_plugin) + + # Sentinel — flips True if get_all_downloads is called on soulseek. + soulseek_called = [] + original_get_all = sl_plugin.get_all_downloads + async def _tracking_get_all(): + soulseek_called.append(True) + return await original_get_all() + sl_plugin.get_all_downloads = _tracking_get_all + + _run_async(engine.get_all_downloads(exclude=('soulseek',))) + assert soulseek_called == [], ( + "soulseek's get_all_downloads was invoked despite being in exclude — " + "monitor would still double-fetch slskd" + ) + + +def test_engine_get_all_downloads_skips_excluded_sources(): + """Per JohnBaumb: monitor pulls slskd transfers via the + transfers/downloads endpoint earlier in its loop, so engine + aggregation must skip soulseek to avoid double-fetching.""" + engine = DownloadEngine() + sl_plugin = _FakePlugin('soulseek', downloads=[_FakeStatus('sl-1', 'soulseek')]) + yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')]) + td_plugin = _FakePlugin('tidal', downloads=[_FakeStatus('td-1', 'tidal')]) + engine.register_plugin('soulseek', sl_plugin) + engine.register_plugin('youtube', yt_plugin) + engine.register_plugin('tidal', td_plugin) + + result = _run_async(engine.get_all_downloads(exclude=('soulseek',))) + ids = {r.id for r in result} + assert ids == {'yt-1', 'td-1'} + assert 'sl-1' not in ids + + +def test_engine_get_all_downloads_swallows_per_plugin_exceptions(): + """One plugin throwing must NOT take down the whole list — same + defensive behavior as the legacy orchestrator (matched by + `try ... except: pass` on every iteration).""" + engine = DownloadEngine() + + class _BrokenPlugin: + async def get_all_downloads(self): + raise RuntimeError("boom") + + yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('yt-1', 'youtube')]) + engine.register_plugin('broken', _BrokenPlugin()) + engine.register_plugin('youtube', yt_plugin) + + result = _run_async(engine.get_all_downloads()) + assert [r.id for r in result] == ['yt-1'] + + +def test_engine_get_download_status_returns_first_match(): + engine = DownloadEngine() + yt_plugin = _FakePlugin('youtube', downloads=[_FakeStatus('shared', 'youtube')]) + td_plugin = _FakePlugin('tidal', downloads=[]) + engine.register_plugin('youtube', yt_plugin) + engine.register_plugin('tidal', td_plugin) + + result = _run_async(engine.get_download_status('shared')) + assert result is not None + assert result.id == 'shared' + + +def test_engine_cancel_routes_streaming_source_directly(): + """When source_hint is a known streaming-source name (not + 'soulseek'), engine routes the cancel to that specific plugin + only — doesn't ask every other plugin first.""" + engine = DownloadEngine() + yt_plugin = _FakePlugin('youtube') + td_plugin = _FakePlugin('tidal') + engine.register_plugin('youtube', yt_plugin) + engine.register_plugin('tidal', td_plugin) + + _run_async(engine.cancel_download('dl-1', 'tidal', remove=False)) + assert yt_plugin.cancel_calls == [] + assert td_plugin.cancel_calls == [('dl-1', 'tidal', False)] + + +def test_engine_cancel_routes_unknown_source_hint_to_soulseek(): + """A username that's NOT in the plugin registry is a real + Soulseek peer name — route to the soulseek plugin.""" + engine = DownloadEngine() + sl_plugin = _FakePlugin('soulseek') + yt_plugin = _FakePlugin('youtube') + engine.register_plugin('soulseek', sl_plugin) + engine.register_plugin('youtube', yt_plugin) + + _run_async(engine.cancel_download('dl-1', 'random_peer_username', remove=False)) + assert sl_plugin.cancel_calls == [('dl-1', 'random_peer_username', False)] + assert yt_plugin.cancel_calls == [] + + +def test_engine_cancel_falls_back_to_iterating_all_plugins_without_hint(): + """No source hint → ask every plugin until one accepts the + cancel (returns True). Mirrors legacy orchestrator behavior.""" + engine = DownloadEngine() + yt_plugin = _FakePlugin('youtube', cancel_result=False) + td_plugin = _FakePlugin('tidal', cancel_result=True) + engine.register_plugin('youtube', yt_plugin) + engine.register_plugin('tidal', td_plugin) + + result = _run_async(engine.cancel_download('dl-1', None, remove=False)) + assert result is True + # Both plugins were asked; tidal accepted. + assert len(yt_plugin.cancel_calls) == 1 + assert len(td_plugin.cancel_calls) == 1 + + +def test_engine_clear_all_skips_unconfigured_plugins(): + """Unconfigured plugins are silently skipped (no API call, no + error) — matches legacy orchestrator's defensive handling.""" + engine = DownloadEngine() + configured = _FakePlugin('youtube', configured=True, clear_result=True) + unconfigured = _FakePlugin('tidal', configured=False) + engine.register_plugin('youtube', configured) + engine.register_plugin('tidal', unconfigured) + + result = _run_async(engine.clear_all_completed_downloads()) + assert result is True + assert configured.clear_calls == 1 + assert unconfigured.clear_calls == 0 + + +def test_engine_clear_all_returns_false_when_any_configured_plugin_fails(): + engine = DownloadEngine() + failing = _FakePlugin('youtube', configured=True, clear_result=False) + engine.register_plugin('youtube', failing) + + result = _run_async(engine.clear_all_completed_downloads()) + assert result is False + + +# --------------------------------------------------------------------------- +# Hybrid fallback (Phase F) +# --------------------------------------------------------------------------- + + +class _FakeSearchPlugin: + def __init__(self, name, configured=True, search_result=None, raises=None): + self.name = name + self._configured = configured + self._search_result = search_result if search_result is not None else ([], []) + self._raises = raises + self.search_calls = 0 + + def is_configured(self): + return self._configured + + async def search(self, query, timeout=None, progress_callback=None): + self.search_calls += 1 + if self._raises: + raise self._raises + return self._search_result + + +class _FakeDownloadPlugin: + def __init__(self, name, configured=True, download_result=None, raises=None): + self.name = name + self._configured = configured + self._download_result = download_result + self._raises = raises + self.download_calls = [] + + def is_configured(self): + return self._configured + + async def download(self, username, filename, file_size): + self.download_calls.append((username, filename, file_size)) + if self._raises: + raise self._raises + return self._download_result + + +def test_search_with_fallback_returns_first_non_empty_result(): + engine = DownloadEngine() + yt = _FakeSearchPlugin('youtube', search_result=([], [])) + td = _FakeSearchPlugin('tidal', search_result=(['track1'], [])) + qz = _FakeSearchPlugin('qobuz', search_result=(['track2'], [])) + engine.register_plugin('youtube', yt) + engine.register_plugin('tidal', td) + engine.register_plugin('qobuz', qz) + + tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal', 'qobuz'])) + assert tracks == ['track1'] + # Tidal short-circuits — qobuz never queried. + assert yt.search_calls == 1 + assert td.search_calls == 1 + assert qz.search_calls == 0 + + +def test_search_with_fallback_skips_unconfigured_plugins(): + engine = DownloadEngine() + yt = _FakeSearchPlugin('youtube', configured=False) + td = _FakeSearchPlugin('tidal', configured=True, search_result=(['hit'], [])) + engine.register_plugin('youtube', yt) + engine.register_plugin('tidal', td) + + tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal'])) + assert tracks == ['hit'] + assert yt.search_calls == 0 # skipped + + +def test_search_with_fallback_continues_after_per_source_exception(): + engine = DownloadEngine() + yt = _FakeSearchPlugin('youtube', raises=RuntimeError("yt down")) + td = _FakeSearchPlugin('tidal', search_result=(['fallback-hit'], [])) + engine.register_plugin('youtube', yt) + engine.register_plugin('tidal', td) + + tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal'])) + assert tracks == ['fallback-hit'] + + +def test_search_with_fallback_returns_empty_when_chain_exhausted(): + engine = DownloadEngine() + yt = _FakeSearchPlugin('youtube', search_result=([], [])) + td = _FakeSearchPlugin('tidal', search_result=([], [])) + engine.register_plugin('youtube', yt) + engine.register_plugin('tidal', td) + + tracks, _ = _run_async(engine.search_with_fallback('q', ['youtube', 'tidal'])) + assert tracks == [] + assert yt.search_calls == 1 + assert td.search_calls == 1 + + +def test_download_with_fallback_returns_first_accepted_download_id(): + """Phase F bug fix: legacy hybrid download routed to one source + via username hint with no retry. Engine now falls through chain.""" + engine = DownloadEngine() + yt = _FakeDownloadPlugin('youtube', download_result=None) # refuses + td = _FakeDownloadPlugin('tidal', download_result='td-id') + engine.register_plugin('youtube', yt) + engine.register_plugin('tidal', td) + + result = _run_async(engine.download_with_fallback( + 'youtube', 'v||t', 0, ['youtube', 'tidal'], + )) + assert result == 'td-id' + assert len(yt.download_calls) == 1 # tried first + assert len(td.download_calls) == 1 # took over + + +def test_download_with_fallback_promotes_username_hint_to_head(): + """A username hint that matches a source-chain entry tries that + source FIRST regardless of declared chain order.""" + engine = DownloadEngine() + yt = _FakeDownloadPlugin('youtube', download_result='yt-id') + td = _FakeDownloadPlugin('tidal', download_result='td-id') + engine.register_plugin('youtube', yt) + engine.register_plugin('tidal', td) + + # Chain says tidal-first, but username hint promotes youtube. + result = _run_async(engine.download_with_fallback( + 'youtube', 'v||t', 0, ['tidal', 'youtube'], + )) + assert result == 'yt-id' + assert len(yt.download_calls) == 1 + assert len(td.download_calls) == 0 # never reached + + +def test_download_with_fallback_returns_none_when_all_refuse(): + engine = DownloadEngine() + yt = _FakeDownloadPlugin('youtube', download_result=None) + td = _FakeDownloadPlugin('tidal', download_result=None) + engine.register_plugin('youtube', yt) + engine.register_plugin('tidal', td) + + result = _run_async(engine.download_with_fallback( + 'youtube', 'v||t', 0, ['youtube', 'tidal'], + )) + assert result is None + assert len(yt.download_calls) == 1 + assert len(td.download_calls) == 1 + + +def test_download_with_fallback_continues_past_exception(): + engine = DownloadEngine() + yt = _FakeDownloadPlugin('youtube', raises=RuntimeError("yt died")) + td = _FakeDownloadPlugin('tidal', download_result='td-id') + engine.register_plugin('youtube', yt) + engine.register_plugin('tidal', td) + + result = _run_async(engine.download_with_fallback( + 'youtube', 'v||t', 0, ['youtube', 'tidal'], + )) + assert result == 'td-id' + + +# --------------------------------------------------------------------------- +# Cin bug 1: alias resolution on cancel_download +# --------------------------------------------------------------------------- + + +def test_register_plugin_records_aliases(): + """Aliases passed to register_plugin resolve to the canonical plugin + via get_plugin. Cin caught engine.cancel_download routing 'deezer_dl' + to soulseek because the alias never made it to the engine.""" + engine = DownloadEngine() + deezer = _FakePlugin('deezer') + engine.register_plugin('deezer', deezer, aliases=('deezer_dl',)) + + assert engine.get_plugin('deezer') is deezer + assert engine.get_plugin('deezer_dl') is deezer + + +def test_cancel_download_resolves_alias_to_canonical_plugin(): + """The legacy 'deezer_dl' source_hint must route to the deezer + plugin, not fall through to soulseek. This was Cin's bug 1 — + cancel of a Deezer download silently no-op'd.""" + engine = DownloadEngine() + soulseek = _FakePlugin('soulseek') + deezer = _FakePlugin('deezer') + engine.register_plugin('soulseek', soulseek) + engine.register_plugin('deezer', deezer, aliases=('deezer_dl',)) + + _run_async(engine.cancel_download('dl-1', 'deezer_dl', remove=False)) + assert deezer.cancel_calls == [('dl-1', 'deezer_dl', False)] + assert soulseek.cancel_calls == [] + + +# --------------------------------------------------------------------------- +# Cin bug 3: atomic update_record_unless_state +# --------------------------------------------------------------------------- + + +def test_update_record_unless_state_applies_when_state_not_blocked(): + engine = DownloadEngine() + engine.add_record('youtube', 'dl-1', {'state': 'InProgress, Downloading'}) + + applied = engine.update_record_unless_state( + 'youtube', 'dl-1', + {'state': 'Completed, Succeeded', 'progress': 100.0}, + skip_if_state_in=('Cancelled',), + ) + assert applied is True + assert engine.get_record('youtube', 'dl-1')['state'] == 'Completed, Succeeded' + assert engine.get_record('youtube', 'dl-1')['progress'] == 100.0 + + +def test_update_record_unless_state_skips_when_state_blocked(): + """A worker-thread terminal write must NOT clobber a Cancelled + state set by the user. Returns False so caller knows the patch + was skipped.""" + engine = DownloadEngine() + engine.add_record('youtube', 'dl-1', {'state': 'Cancelled'}) + + applied = engine.update_record_unless_state( + 'youtube', 'dl-1', + {'state': 'Completed, Succeeded'}, + skip_if_state_in=('Cancelled',), + ) + assert applied is False + assert engine.get_record('youtube', 'dl-1')['state'] == 'Cancelled' + + +def test_update_record_unless_state_returns_false_for_missing_record(): + engine = DownloadEngine() + applied = engine.update_record_unless_state( + 'youtube', 'never-existed', + {'state': 'Completed, Succeeded'}, + skip_if_state_in=('Cancelled',), + ) + assert applied is False diff --git a/tests/downloads/test_download_orchestrator.py b/tests/downloads/test_download_orchestrator.py index 5e06376c..e157047e 100644 --- a/tests/downloads/test_download_orchestrator.py +++ b/tests/downloads/test_download_orchestrator.py @@ -1,4 +1,6 @@ +from core.download_engine import DownloadEngine from core.download_orchestrator import DownloadOrchestrator +from core.download_plugins.registry import DownloadPluginRegistry, PluginSpec class _FakeClient: @@ -16,14 +18,45 @@ class _FakeClient: def _build_orchestrator(**clients): + """Build an orchestrator with mock clients via the registry. + + The orchestrator iterates `self.registry.all_plugins()` to drive + every per-source operation, so the test must set up a real + registry with mock plugins (not just stuff attributes on the + orchestrator). Source slots not provided in `clients` are + skipped — registry only holds the ones the test cares about. + """ + registry = DownloadPluginRegistry() + name_to_display = { + 'soulseek': 'Soulseek', 'youtube': 'YouTube', 'tidal': 'Tidal', + 'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer', + 'lidarr': 'Lidarr', 'soundcloud': 'SoundCloud', + } + # 'deezer_dl' is the legacy attr name; canonical registry name is 'deezer'. + aliases_for = {'deezer_dl': ('deezer_dl',)} + canonical_for = {'deezer_dl': 'deezer'} + + for slot, client in clients.items(): + if client is None: + continue + canonical_name = canonical_for.get(slot, slot) + registry.register(PluginSpec( + name=canonical_name, + factory=lambda c=client: c, + display_name=name_to_display.get(slot, slot), + aliases=aliases_for.get(slot, ()), + )) + registry.initialize() + orch = DownloadOrchestrator.__new__(DownloadOrchestrator) - orch.soulseek = clients.get("soulseek") - orch.youtube = clients.get("youtube") - orch.tidal = clients.get("tidal") - orch.qobuz = clients.get("qobuz") - orch.hifi = clients.get("hifi") - orch.deezer_dl = clients.get("deezer_dl") - orch.lidarr = clients.get("lidarr") + orch.registry = registry + orch._init_failures = registry.init_failures + # Engine — orchestrator delegates per-source query/cancel + # methods to it, so the test fixture must build one and + # register every mock plugin under its canonical name. + orch.engine = DownloadEngine() + for source_name, plugin in registry.all_plugins(): + orch.engine.register_plugin(source_name, plugin) return orch @@ -46,8 +79,8 @@ def test_clear_all_completed_downloads_ignores_unconfigured_clients(): result = _run_async(orch.clear_all_completed_downloads()) assert result is True - assert orch.soulseek.clear_calls == 1 - assert orch.youtube.clear_calls == 0 + assert orch.client('soulseek').clear_calls == 1 + assert orch.client('youtube').clear_calls == 0 def test_clear_all_completed_downloads_propagates_configured_failures(): @@ -58,4 +91,177 @@ def test_clear_all_completed_downloads_propagates_configured_failures(): result = _run_async(orch.clear_all_completed_downloads()) assert result is False - assert orch.soulseek.clear_calls == 1 + assert orch.client('soulseek').clear_calls == 1 + + +# --------------------------------------------------------------------------- +# Cin-2 generic accessors +# --------------------------------------------------------------------------- + + +def test_client_returns_registered_client_by_name(): + """Cin's review feedback: orch.client('hifi') is the canonical + way to reach a per-source client, replacing orch.hifi attribute + access.""" + soulseek = _FakeClient() + youtube = _FakeClient() + orch = _build_orchestrator(soulseek=soulseek, youtube=youtube) + + assert orch.client('soulseek') is soulseek + assert orch.client('youtube') is youtube + assert orch.client('made_up') is None + + +def test_configured_clients_excludes_unconfigured_sources(): + """Replaces the legacy iteration pattern: 6+ if/hasattr/is_configured + checks per source. Single call returns dict of configured clients.""" + configured = _FakeClient(configured=True) + unconfigured = _FakeClient(configured=False) + orch = _build_orchestrator( + soulseek=configured, + youtube=unconfigured, + ) + result = orch.configured_clients() + assert 'soulseek' in result + assert 'youtube' not in result + assert result['soulseek'] is configured + + +def test_configured_clients_skips_clients_whose_is_configured_raises(): + """Per JohnBaumb: configured_clients() has a try/except so a single + broken is_configured() call doesn't crash the whole iteration — + pin it so a future refactor can't quietly drop the guard. The + broken plugin is skipped; the rest still come back.""" + + class _BrokenIsConfigured(_FakeClient): + def is_configured(self): + raise RuntimeError("is_configured blew up") + + broken = _BrokenIsConfigured() + healthy = _FakeClient(configured=True) + orch = _build_orchestrator(soulseek=healthy, youtube=broken) + + result = orch.configured_clients() + # Healthy plugin still surfaces; broken one is silently skipped. + assert 'soulseek' in result + assert result['soulseek'] is healthy + assert 'youtube' not in result + + +def test_reload_instances_dispatches_to_named_source(): + """Generic dispatch — caller passes source name instead of + reaching for orch.hifi.reload_instances() directly.""" + + class _ReloadableClient(_FakeClient): + def __init__(self): + super().__init__(configured=True) + self.reload_called = False + + def reload_instances(self): + self.reload_called = True + + hifi = _ReloadableClient() + soulseek = _FakeClient() # No reload_instances method + orch = _build_orchestrator(soulseek=soulseek, hifi=hifi) + + assert orch.reload_instances('hifi') is True + assert hifi.reload_called is True + + +def test_reload_instances_skips_clients_without_method(): + """Sources that don't expose reload_instances are skipped, not + treated as failures.""" + soulseek = _FakeClient() # No reload_instances method + orch = _build_orchestrator(soulseek=soulseek) + # Calling on a source without the method = silent no-op + assert orch.reload_instances('soulseek') is True + + +def test_reload_instances_with_no_args_reloads_every_source(): + """When called with no source argument, hits every registered + source that exposes reload_instances.""" + + class _ReloadableClient(_FakeClient): + def __init__(self): + super().__init__() + self.reload_called = False + + def reload_instances(self): + self.reload_called = True + + a = _ReloadableClient() + b = _ReloadableClient() + orch = _build_orchestrator(soulseek=a, hifi=b) + + orch.reload_instances() + assert a.reload_called is True + assert b.reload_called is True + + +# --------------------------------------------------------------------------- +# Singleton factory (matches Cin's get_metadata_engine pattern) +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Cin bug 2: hybrid_order alias normalization +# --------------------------------------------------------------------------- + + +def test_resolve_source_chain_normalizes_legacy_aliases(): + """Cin's bug 2: hybrid_order config containing the legacy alias + 'deezer_dl' was silently dropped because the canonical-name + membership check rejected it. Orchestrator must normalize via + the registry alias map first.""" + orch = _build_orchestrator( + soulseek=_FakeClient(), + deezer_dl=_FakeClient(), + youtube=_FakeClient(), + ) + orch.hybrid_order = ['deezer_dl', 'soulseek', 'youtube'] + orch.hybrid_primary = None + orch.hybrid_secondary = None + + chain = orch._resolve_source_chain() + assert chain == ['deezer', 'soulseek', 'youtube'] + + +def test_resolve_source_chain_dedupes_alias_and_canonical(): + """If both 'deezer' and 'deezer_dl' appear, dedupe to single entry.""" + orch = _build_orchestrator( + soulseek=_FakeClient(), + deezer_dl=_FakeClient(), + ) + orch.hybrid_order = ['deezer_dl', 'deezer', 'soulseek'] + orch.hybrid_primary = None + orch.hybrid_secondary = None + + chain = orch._resolve_source_chain() + assert chain == ['deezer', 'soulseek'] + + +def test_resolve_source_chain_drops_unknown_names(): + orch = _build_orchestrator(soulseek=_FakeClient(), youtube=_FakeClient()) + orch.hybrid_order = ['nonsense', 'soulseek', 'also_fake', 'youtube'] + orch.hybrid_primary = None + orch.hybrid_secondary = None + + chain = orch._resolve_source_chain() + assert chain == ['soulseek', 'youtube'] + + +def test_get_download_orchestrator_returns_set_singleton(): + """When set_download_orchestrator has been called (web_server.py + does this at boot), get_download_orchestrator returns the + installed instance instead of building a fresh one.""" + from core.download_orchestrator import ( + get_download_orchestrator, + set_download_orchestrator, + ) + + orch = _build_orchestrator(soulseek=_FakeClient()) + set_download_orchestrator(orch) + try: + assert get_download_orchestrator() is orch + finally: + set_download_orchestrator(None) diff --git a/tests/downloads/test_downloads_candidates.py b/tests/downloads/test_downloads_candidates.py index bddf3df4..19471384 100644 --- a/tests/downloads/test_downloads_candidates.py +++ b/tests/downloads/test_downloads_candidates.py @@ -101,7 +101,7 @@ def _build_deps( on_complete=None, ): deps = dc.CandidatesDeps( - soulseek_client=soulseek or _FakeSoulseek(), + download_orchestrator=soulseek or _FakeSoulseek(), spotify_client=spotify or _FakeSpotify(), run_async=_run_async, get_database=lambda: db or _FakeDB(), @@ -136,7 +136,7 @@ def test_first_candidate_starts_download_and_returns_true(): result = dc.attempt_download_with_candidates("t1", candidates, track, batch_id="b1", deps=deps) assert result is True - assert deps.soulseek_client.download_calls == [("user1", "best.flac", 1000)] + assert deps.download_orchestrator.download_calls == [("user1", "best.flac", 1000)] assert download_tasks["t1"]["download_id"] == "dl-1" assert "user1::best.flac" in matched_downloads_context @@ -155,7 +155,7 @@ def test_candidates_tried_in_confidence_order(): dc.attempt_download_with_candidates("t2", candidates, track, batch_id=None, deps=deps) # First call should be the highest-confidence one - assert deps.soulseek_client.download_calls[0][1] == "high.flac" + assert deps.download_orchestrator.download_calls[0][1] == "high.flac" # --------------------------------------------------------------------------- @@ -175,8 +175,8 @@ def test_already_tried_source_skipped(): dc.attempt_download_with_candidates("t3", candidates, track, batch_id=None, deps=deps) # First candidate skipped (already used), second one tried - assert len(deps.soulseek_client.download_calls) == 1 - assert deps.soulseek_client.download_calls[0][1] == "fresh.flac" + assert len(deps.download_orchestrator.download_calls) == 1 + assert deps.download_orchestrator.download_calls[0][1] == "fresh.flac" # --------------------------------------------------------------------------- @@ -196,7 +196,7 @@ def test_blacklisted_source_skipped(): dc.attempt_download_with_candidates("t4", candidates, track, batch_id=None, deps=deps) - assert deps.soulseek_client.download_calls[0][1] == "ok.flac" + assert deps.download_orchestrator.download_calls[0][1] == "ok.flac" # --------------------------------------------------------------------------- @@ -213,7 +213,7 @@ def test_cancellation_before_attempt_returns_false(): result = dc.attempt_download_with_candidates("t5", candidates, track, batch_id=None, deps=deps) assert result is False - assert deps.soulseek_client.download_calls == [] + assert deps.download_orchestrator.download_calls == [] def test_task_deleted_returns_false(): @@ -238,7 +238,7 @@ def test_active_download_id_skips_new_download(): dc.attempt_download_with_candidates("t6", candidates, track, batch_id=None, deps=deps) # Both candidates skipped (download_id already present) - assert deps.soulseek_client.download_calls == [] + assert deps.download_orchestrator.download_calls == [] def test_cancellation_after_download_starts_calls_cancel_and_lifecycle(): @@ -266,7 +266,7 @@ def test_cancellation_after_download_starts_calls_cancel_and_lifecycle(): assert result is False # cancel_download was called for the in-flight transfer - assert deps.soulseek_client.cancel_calls + assert deps.download_orchestrator.cancel_calls # on_download_completed fired with success=False to free the worker slot assert completion_calls == [("b7", "t7", False)] @@ -276,7 +276,7 @@ def test_cancellation_after_download_starts_calls_cancel_and_lifecycle(): # --------------------------------------------------------------------------- def test_all_candidates_failed_returns_false(): - """If soulseek_client.download returns None (failure) for all candidates, returns False.""" + """If download_orchestrator.download returns None (failure) for all candidates, returns False.""" soulseek = _FakeSoulseek(download_id=None) deps = _build_deps(soulseek=soulseek) _seed_task("t8") @@ -411,5 +411,5 @@ def test_candidates_with_equal_confidence_both_tried(): dc.attempt_download_with_candidates("t13", candidates, track, batch_id=None, deps=deps) # First one wins — second never tried because download succeeded - assert len(deps.soulseek_client.download_calls) == 1 - assert deps.soulseek_client.download_calls[0][1] == "a.flac" + assert len(deps.download_orchestrator.download_calls) == 1 + assert deps.download_orchestrator.download_calls[0][1] == "a.flac" diff --git a/tests/downloads/test_downloads_master.py b/tests/downloads/test_downloads_master.py index 5307dd2f..e702e71e 100644 --- a/tests/downloads/test_downloads_master.py +++ b/tests/downloads/test_downloads_master.py @@ -178,7 +178,7 @@ def _build_deps( ): return mw.MasterDeps( config_manager=config or _FakeConfig(), - soulseek_client=soulseek or _FakeSoulseekWrapper(_FakeSoulseek()), + download_orchestrator=soulseek or _FakeSoulseekWrapper(_FakeSoulseek()), run_async=run_async or _make_run_async(), mb_worker=mb_worker, mb_release_cache=mb_release_cache if mb_release_cache is not None else {}, diff --git a/tests/downloads/test_downloads_post_processing.py b/tests/downloads/test_downloads_post_processing.py index 2076ef6e..a5f02d8c 100644 --- a/tests/downloads/test_downloads_post_processing.py +++ b/tests/downloads/test_downloads_post_processing.py @@ -49,7 +49,7 @@ class _Recorder: def _build_deps( *, config=None, - soulseek_client=None, + download_orchestrator=None, run_async=None, docker_resolve_path=None, extract_filename=None, @@ -64,7 +64,7 @@ def _build_deps( rec = _Recorder() return pp.PostProcessDeps( config_manager=config or _FakeConfig(), - soulseek_client=soulseek_client, + download_orchestrator=download_orchestrator, run_async=run_async or (lambda c: None), docker_resolve_path=docker_resolve_path or (lambda p: p), extract_filename=extract_filename or (lambda f: os.path.basename(f) if f else ''), @@ -315,7 +315,7 @@ def test_youtube_task_uses_get_download_status_to_resolve_path(monkeypatch): monkeypatch.setattr(pp.os.path, 'exists', lambda p: p == '/downloads/Money.mp3') deps, rec = _build_deps( - soulseek_client=_FakeYTClient(), + download_orchestrator=_FakeYTClient(), run_async=lambda coro: coro, # not async — direct call ) pp.run_post_processing_worker('t1', 'b1', deps) diff --git a/tests/downloads/test_downloads_task_worker.py b/tests/downloads/test_downloads_task_worker.py index fca7b10b..11ccdfca 100644 --- a/tests/downloads/test_downloads_task_worker.py +++ b/tests/downloads/test_downloads_task_worker.py @@ -31,13 +31,28 @@ class _Recorder: class _FakeClient: - """Stub soulseek client. `mode` defaults to non-hybrid.""" + """Stub download orchestrator. `mode` defaults to non-hybrid. + Per-source stubs passed via ``subclients`` are surfaced through + ``client(name)`` (the registry-backed accessor on the real + orchestrator); plain attrs (hybrid_order, hybrid_primary, etc.) + are set as attributes so getattr() lookups still resolve them.""" + _CLIENT_NAMES = {'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', + 'deezer_dl', 'lidarr', 'soundcloud'} + def __init__(self, results=None, mode='soulseek', subclients=None): self._results = results if results is not None else [] self.mode = mode self.search_calls = [] + self._client_map = {} for k, v in (subclients or {}).items(): - setattr(self, k, v) + if k in self._CLIENT_NAMES: + self._client_map[k] = v + else: + # config-style attrs (hybrid_order, hybrid_primary, etc.) + setattr(self, k, v) + + def client(self, name): + return self._client_map.get(name) async def search(self, query, timeout=30): self.search_calls.append((query, timeout)) @@ -76,7 +91,7 @@ def _build_deps( ): rec = _Recorder() return tw.TaskWorkerDeps( - soulseek_client=soulseek or _FakeClient(), + download_orchestrator=soulseek or _FakeClient(), matching_engine=matching or _FakeMatchEngine(), run_async=_sync_run_async, try_source_reuse=try_source_reuse, diff --git a/tests/downloads/test_downloads_validation.py b/tests/downloads/test_downloads_validation.py new file mode 100644 index 00000000..62798523 --- /dev/null +++ b/tests/downloads/test_downloads_validation.py @@ -0,0 +1,92 @@ +"""Tests for core/downloads/validation.py — SoundCloud preview filter. + +The SoundCloud anonymous tier serves a ~30s preview clip for tracks +gated behind Go+ / login. ``filter_soundcloud_previews`` drops these +candidates before they reach the matcher, the modal cache, or the +manual-pick download path. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from core.downloads.validation import filter_soundcloud_previews + + +@dataclass +class _Track: + duration_ms: int + + +@dataclass +class _Candidate: + username: str + duration: Optional[int] # milliseconds + title: str = '' + + +def test_drops_soundcloud_30s_preview_when_expected_long(): + """A 30s SC candidate against a 5-minute expected track is the + canonical preview-snippet case — must be dropped.""" + expected = _Track(duration_ms=338_000) # ~5:38 + cands = [ + _Candidate(username='soundcloud', duration=30_000, title='Preview'), + _Candidate(username='soundcloud', duration=338_000, title='Real'), + ] + result = filter_soundcloud_previews(cands, expected) + assert len(result) == 1 + assert result[0].title == 'Real' + + +def test_drops_under_half_expected_duration(): + """SC candidate at 100s against 300s expected = clearly truncated / + wrong content. Must be dropped even if not at the 30s boundary.""" + expected = _Track(duration_ms=300_000) + cand = _Candidate(username='soundcloud', duration=100_000) + assert filter_soundcloud_previews([cand], expected) == [] + + +def test_keeps_soundcloud_when_expected_track_is_short(): + """Genuinely short SC tracks (intros, sound effects, sub-minute + songs) must pass through when the expected track is also short. + Filter only kicks in when expected > 60s.""" + expected = _Track(duration_ms=45_000) # 45s expected + cand = _Candidate(username='soundcloud', duration=30_000) + result = filter_soundcloud_previews([cand], expected) + assert result == [cand] + + +def test_does_not_filter_non_soundcloud_sources(): + """A 30s candidate from another streaming source isn't a SoundCloud + preview — leave it for the generic matching engine to score.""" + expected = _Track(duration_ms=338_000) + yt = _Candidate(username='youtube', duration=30_000) + tidal = _Candidate(username='tidal', duration=30_000) + assert filter_soundcloud_previews([yt, tidal], expected) == [yt, tidal] + + +def test_returns_input_unchanged_without_expected_duration(): + """Without a Spotify-track / expected duration we can't reason + about previews — pass everything through.""" + cands = [ + _Candidate(username='soundcloud', duration=30_000), + _Candidate(username='soundcloud', duration=300_000), + ] + assert filter_soundcloud_previews(cands, None) == cands + assert filter_soundcloud_previews(cands, _Track(duration_ms=0)) == cands + + +def test_empty_input_returns_empty_list(): + assert filter_soundcloud_previews([], _Track(duration_ms=200_000)) == [] + + +def test_keeps_soundcloud_candidate_at_threshold(): + """Boundary check: 35s candidate against 200s expected — exactly + at the 35s preview boundary, but 35s is also above + expected*0.5 (100s) check (35 < 100, so still drops). Use a + higher value to confirm the just-above threshold passes.""" + expected = _Track(duration_ms=200_000) # 200s + # 110s passes both checks: > 35s AND > 100s (half of 200s) + cand = _Candidate(username='soundcloud', duration=110_000) + assert filter_soundcloud_previews([cand], expected) == [cand] diff --git a/tests/downloads/test_hifi_pinning.py b/tests/downloads/test_hifi_pinning.py new file mode 100644 index 00000000..24ca15f1 --- /dev/null +++ b/tests/downloads/test_hifi_pinning.py @@ -0,0 +1,104 @@ +"""Phase A pinning tests for HiFiClient — UPDATED for Phase C5.""" + +from __future__ import annotations + +import asyncio +import threading +from pathlib import Path +from unittest.mock import patch + +import pytest + +from core.download_engine import DownloadEngine +from core.hifi_client import HiFiClient + + +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture +def hifi_client_with_engine(): + client = HiFiClient.__new__(HiFiClient) + client.download_path = Path('./test_hifi_downloads') + client.shutdown_check = None + client._engine = None + engine = DownloadEngine() + client.set_engine(engine) + return client, engine + + +def test_download_returns_none_for_invalid_filename_format(hifi_client_with_engine): + client, _ = hifi_client_with_engine + result = _run_async(client.download('hifi', 'no-separator', 0)) + assert result is None + + +def test_download_returns_none_for_non_integer_track_id(hifi_client_with_engine): + client, _ = hifi_client_with_engine + result = _run_async(client.download('hifi', 'not-int||title', 0)) + assert result is None + + +def test_download_raises_when_engine_not_wired(): + """Defensive: client without engine reference must raise so the + orchestrator's download_with_fallback surfaces the error and + moves on to the next source. Returning None silently would drop + the download with no user feedback (per JohnBaumb).""" + import pytest + client = HiFiClient.__new__(HiFiClient) + client._engine = None + with pytest.raises(RuntimeError, match="engine reference"): + _run_async(client.download('hifi', 'v||t', 0)) + + +def test_download_returns_uuid_for_valid_filename(hifi_client_with_engine): + client, _ = hifi_client_with_engine + with patch.object(client, '_download_sync', return_value='/tmp/x.flac'): + result = _run_async(client.download('hifi', '12345||Some Song', 0)) + assert result is not None + assert len(result) == 36 + + +def test_download_populates_engine_record_with_initial_state(hifi_client_with_engine): + client, engine = hifi_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/done.flac' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download('hifi', '999||My HiFi Song', 0)) + started.wait(timeout=1.0) + record = engine.get_record('hifi', download_id) + assert record['filename'] == '999||My HiFi Song' + assert record['username'] == 'hifi' + assert record['track_id'] == 999 + assert record['display_name'] == 'My HiFi Song' + release.set() + + +def test_get_all_downloads_reads_engine_records(hifi_client_with_engine): + client, engine = hifi_client_with_engine + engine.add_record('hifi', 'dl-1', { + 'id': 'dl-1', 'filename': '111||A', 'username': 'hifi', + 'state': 'InProgress, Downloading', 'progress': 50.0, + }) + result = _run_async(client.get_all_downloads()) + assert len(result) == 1 + assert result[0].id == 'dl-1' + + +def test_cancel_download_marks_cancelled(hifi_client_with_engine): + client, engine = hifi_client_with_engine + engine.add_record('hifi', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'}) + ok = _run_async(client.cancel_download('dl-1', None, remove=False)) + assert ok is True + assert engine.get_record('hifi', 'dl-1')['state'] == 'Cancelled' diff --git a/tests/downloads/test_lidarr_pinning.py b/tests/downloads/test_lidarr_pinning.py new file mode 100644 index 00000000..2eb88738 --- /dev/null +++ b/tests/downloads/test_lidarr_pinning.py @@ -0,0 +1,138 @@ +"""Phase A pinning tests for LidarrDownloadClient's download lifecycle. + +Lidarr is the special case in the dispatcher — it's an +ALBUM-grabber, not a track-grabber. When the user asks for a +track, Lidarr grabs the whole album, then we pick the wanted +track out (logic at the end of `_download_thread_worker`). + +Engine refactor's plugin contract must accommodate album-only +sources OR Lidarr stays special. Pinning the current contract +forces the design decision to be conscious during Phase G. +""" + +from __future__ import annotations + +import asyncio +import threading +from pathlib import Path +from unittest.mock import patch + +import pytest + +from core.lidarr_download_client import LidarrDownloadClient + + +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture +def lidarr_client(): + client = LidarrDownloadClient.__new__(LidarrDownloadClient) + client.download_path = Path('./test_lidarr_downloads') + client.shutdown_check = None + client.active_downloads = {} + client._download_lock = threading.Lock() + client._url = 'http://lidarr.test' + client._api_key = 'test-key' + return client + + +def test_download_returns_none_when_not_configured(): + """Pinning: no Lidarr URL/key → None. Orchestrator hybrid skip + behavior depends on this.""" + client = LidarrDownloadClient.__new__(LidarrDownloadClient) + client._url = '' + client._api_key = '' + result = _run_async(client.download('lidarr', '12345||Album Name', 0)) + assert result is None + + +def test_download_returns_uuid_for_valid_filename(lidarr_client): + """Pinning: valid filename → UUID download_id.""" + with patch('core.lidarr_download_client.threading.Thread') as fake: + fake.return_value.start = lambda: None + result = _run_async(lidarr_client.download( + 'lidarr', '12345||Some Album', 0, + )) + assert result is not None + assert len(result) == 36 + + +def test_download_parses_album_foreign_id_from_filename(lidarr_client): + """Pinning: filename format is ``album_foreign_id||display`` where + `album_foreign_id` is the MusicBrainz album MBID Lidarr lookups + use. Engine refactor's plugin contract must respect that this + is an ALBUM identifier, not a track.""" + with patch('core.lidarr_download_client.threading.Thread') as fake: + fake.return_value.start = lambda: None + download_id = _run_async(lidarr_client.download( + 'lidarr', 'mbid-album-123||Some Album by Artist', 0, + )) + + record = lidarr_client.active_downloads[download_id] + assert record['album_foreign_id'] == 'mbid-album-123' + assert record['display_name'] == 'Some Album by Artist' + + +def test_download_handles_filename_without_separator(lidarr_client): + """Pinning: defensive — filename without `||` still produces a + download record (album_foreign_id stays empty, display_name is + the whole filename). Lidarr's worker tries lookup-by-display.""" + with patch('core.lidarr_download_client.threading.Thread') as fake: + fake.return_value.start = lambda: None + download_id = _run_async(lidarr_client.download( + 'lidarr', 'just-some-display-name', 0, + )) + + assert download_id is not None + record = lidarr_client.active_downloads[download_id] + assert record['album_foreign_id'] == '' + assert record['display_name'] == 'just-some-display-name' + + +def test_download_populates_active_downloads_with_album_oriented_state(lidarr_client): + """Pinning: Lidarr's state-dict is SMALLER than streaming sources + (no track_id, no transferred/speed/time_remaining — Lidarr + polls Lidarr's queue API for those, doesn't track byte-level + progress locally). Engine extraction must accommodate the + smaller schema.""" + with patch('core.lidarr_download_client.threading.Thread') as fake: + fake.return_value.start = lambda: None + download_id = _run_async(lidarr_client.download( + 'lidarr', 'mbid-1||Album', 0, + )) + + record = lidarr_client.active_downloads[download_id] + assert record['id'] == download_id + assert record['username'] == 'lidarr' + assert record['state'] == 'Initializing' + assert record['progress'] == 0.0 + assert record['album_foreign_id'] == 'mbid-1' + assert record['file_path'] is None + + +def test_download_spawns_daemon_thread_targeting_worker(lidarr_client): + """Pinning: thread target is `_download_thread_worker(download_id, + album_foreign_id, display_name)` — 3 args, not 4 like streaming + sources. Lidarr doesn't need original_filename because the album + foreign id IS the unique key.""" + captured_kwargs = {} + + def capture_thread(*args, **kwargs): + captured_kwargs.update(kwargs) + return type('FakeThread', (), {'start': lambda self: None})() + + with patch('core.lidarr_download_client.threading.Thread', side_effect=capture_thread): + _run_async(lidarr_client.download('lidarr', 'mbid-x||Album', 0)) + + assert captured_kwargs.get('daemon') is True + assert captured_kwargs.get('target') == lidarr_client._download_thread_worker + args = captured_kwargs.get('args', ()) + assert len(args) == 3 # 3-arg signature + assert args[1] == 'mbid-x' + assert args[2] == 'Album' diff --git a/tests/downloads/test_qobuz_pinning.py b/tests/downloads/test_qobuz_pinning.py new file mode 100644 index 00000000..ab55d8ed --- /dev/null +++ b/tests/downloads/test_qobuz_pinning.py @@ -0,0 +1,121 @@ +"""Phase A pinning tests for QobuzClient — UPDATED for Phase C4. + +Post-C4 the client uses engine.worker for thread + state management. +""" + +from __future__ import annotations + +import asyncio +import threading +from pathlib import Path +from unittest.mock import patch + +import pytest + +from core.download_engine import DownloadEngine +from core.qobuz_client import QobuzClient + + +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture +def qobuz_client_with_engine(): + client = QobuzClient.__new__(QobuzClient) + client.download_path = Path('./test_qobuz_downloads') + client.shutdown_check = None + client._engine = None + engine = DownloadEngine() + client.set_engine(engine) + return client, engine + + +def test_download_returns_none_for_invalid_filename_format(qobuz_client_with_engine): + client, _ = qobuz_client_with_engine + result = _run_async(client.download('qobuz', 'no-separator', 0)) + assert result is None + + +def test_download_returns_none_for_non_integer_track_id(qobuz_client_with_engine): + client, _ = qobuz_client_with_engine + result = _run_async(client.download('qobuz', 'not-int||title', 0)) + assert result is None + + +def test_download_raises_when_engine_not_wired(): + """Defensive: client without engine reference must raise so the + orchestrator's download_with_fallback surfaces the error and + moves on to the next source. Returning None silently would drop + the download with no user feedback (per JohnBaumb).""" + import pytest + client = QobuzClient.__new__(QobuzClient) + client._engine = None + with pytest.raises(RuntimeError, match="engine reference"): + _run_async(client.download('qobuz', 'v||t', 0)) + + +def test_download_returns_uuid_for_valid_filename(qobuz_client_with_engine): + client, _ = qobuz_client_with_engine + with patch.object(client, '_download_sync', return_value='/tmp/x.flac'): + result = _run_async(client.download('qobuz', '12345||Some Song', 0)) + assert result is not None + assert len(result) == 36 + + +def test_download_populates_engine_record_with_initial_state(qobuz_client_with_engine): + client, engine = qobuz_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/done.flac' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download('qobuz', '999||My Qobuz Song', 0)) + started.wait(timeout=1.0) + record = engine.get_record('qobuz', download_id) + + assert record is not None + assert record['filename'] == '999||My Qobuz Song' + assert record['username'] == 'qobuz' + assert record['track_id'] == 999 + assert record['display_name'] == 'My Qobuz Song' + release.set() + + +def test_get_all_downloads_reads_engine_records(qobuz_client_with_engine): + client, engine = qobuz_client_with_engine + engine.add_record('qobuz', 'dl-1', { + 'id': 'dl-1', 'filename': '111||A', 'username': 'qobuz', + 'state': 'InProgress, Downloading', 'progress': 50.0, + }) + result = _run_async(client.get_all_downloads()) + assert len(result) == 1 + assert result[0].id == 'dl-1' + + +def test_cancel_download_marks_cancelled(qobuz_client_with_engine): + client, engine = qobuz_client_with_engine + engine.add_record('qobuz', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'}) + + ok = _run_async(client.cancel_download('dl-1', None, remove=False)) + assert ok is True + assert engine.get_record('qobuz', 'dl-1')['state'] == 'Cancelled' + + +def test_clear_all_completed_drops_only_terminal_records(qobuz_client_with_engine): + client, engine = qobuz_client_with_engine + engine.add_record('qobuz', 'done', {'id': 'done', 'state': 'Completed, Succeeded'}) + engine.add_record('qobuz', 'live', {'id': 'live', 'state': 'InProgress, Downloading'}) + + _run_async(client.clear_all_completed_downloads()) + + assert engine.get_record('qobuz', 'done') is None + assert engine.get_record('qobuz', 'live') is not None diff --git a/tests/downloads/test_rate_limit_policy.py b/tests/downloads/test_rate_limit_policy.py new file mode 100644 index 00000000..ec454952 --- /dev/null +++ b/tests/downloads/test_rate_limit_policy.py @@ -0,0 +1,105 @@ +"""Tests for the per-source RateLimitPolicy declaration mechanism (Phase E1).""" + +from __future__ import annotations + +from core.download_engine import DownloadEngine, RateLimitPolicy +from core.download_engine.rate_limit import DEFAULT_POLICY, resolve_policy + + +# --------------------------------------------------------------------------- +# resolve_policy +# --------------------------------------------------------------------------- + + +def test_resolve_policy_returns_default_when_plugin_declares_nothing(): + plugin = object() + assert resolve_policy(plugin) is DEFAULT_POLICY + + +def test_resolve_policy_reads_class_attribute(): + class _Plugin: + RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=5.0) + + policy = resolve_policy(_Plugin()) + assert policy.download_delay_seconds == 5.0 + + +def test_resolve_policy_prefers_method_over_class_attribute(): + class _Plugin: + RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=1.0) + + def rate_limit_policy(self): + return RateLimitPolicy(download_delay_seconds=10.0) + + assert resolve_policy(_Plugin()).download_delay_seconds == 10.0 + + +def test_resolve_policy_falls_back_to_default_when_method_returns_garbage(): + class _Plugin: + def rate_limit_policy(self): + return "not a policy object" + + assert resolve_policy(_Plugin()) is DEFAULT_POLICY + + +def test_resolve_policy_falls_back_to_default_when_method_raises(): + class _Plugin: + def rate_limit_policy(self): + raise RuntimeError("boom") + + assert resolve_policy(_Plugin()) is DEFAULT_POLICY + + +# --------------------------------------------------------------------------- +# Engine applies policy on register +# --------------------------------------------------------------------------- + + +def test_engine_applies_declared_policy_on_register(): + """Pinning: when a plugin is registered, its declared + RateLimitPolicy is pushed into the worker's per-source semaphore + + delay registry. Future dispatches use those values.""" + class _ThrottledPlugin: + RATE_LIMIT_POLICY = RateLimitPolicy(download_concurrency=1, download_delay_seconds=2.5) + + engine = DownloadEngine() + engine.register_plugin('throttled', _ThrottledPlugin()) + + assert engine.worker._get_delay('throttled') == 2.5 + + +def test_engine_applies_default_policy_when_plugin_declares_nothing(): + """Plugins without a declaration get the conservative default + (concurrency=1, delay=0).""" + class _DefaultPlugin: + pass + + engine = DownloadEngine() + engine.register_plugin('default', _DefaultPlugin()) + + assert engine.worker._get_delay('default') == 0.0 + + +def test_set_engine_callback_runs_after_policy_applied(): + """Pinning: set_engine fires AFTER policy registration, so + config-driven sources can override their declared policy. + YouTube uses this — set_engine reads the user-tunable + youtube.download_delay config and overrides the declared default.""" + fired_at: list = [] + + class _Plugin: + RATE_LIMIT_POLICY = RateLimitPolicy(download_delay_seconds=1.0) + + def set_engine(self, engine): + # Capture worker state at the moment set_engine fires. + fired_at.append(engine.worker._get_delay('flexible')) + # Then override. + engine.worker.set_delay('flexible', 99.0) + + engine = DownloadEngine() + engine.register_plugin('flexible', _Plugin()) + + # The class-attribute value was applied first. + assert fired_at == [1.0] + # Then set_engine overrode it. + assert engine.worker._get_delay('flexible') == 99.0 diff --git a/tests/downloads/test_soulseek_pinning.py b/tests/downloads/test_soulseek_pinning.py new file mode 100644 index 00000000..253be224 --- /dev/null +++ b/tests/downloads/test_soulseek_pinning.py @@ -0,0 +1,349 @@ +"""Phase A pinning tests for SoulseekClient's download lifecycle. + +These tests pin the OBSERVABLE BEHAVIOR of `SoulseekClient.download` / +`get_all_downloads` / `cancel_download` so the upcoming download +engine refactor (which lifts shared state + thread workers + search +retry into a central engine) can't drift the per-source contract. + +The contract these tests pin is what the engine will call into via +`plugin.download_raw(target_id)` / `plugin.cancel_raw(target_id)` +after the refactor lands. If a future commit breaks any of these +expectations, the diff fails fast — long before a real download +attempt against a live slskd would have surfaced the bug. + +NOTE: Soulseek is structurally different from the streaming sources. +It has NO local thread worker — slskd manages downloads server-side +and the client just polls for state. So Soulseek skips most of the +engine refactor's thread-extraction work; what stays critical is +the slskd HTTP API contract (endpoints, payload shape, id +extraction). That's what these tests pin. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest + +from core.soulseek_client import SoulseekClient + + +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +# --------------------------------------------------------------------------- +# Configuration / lifecycle +# --------------------------------------------------------------------------- + + +def test_is_configured_returns_false_when_no_base_url(): + """Pinning: an unconfigured client (no slskd URL set) reports + is_configured() == False. The orchestrator's hybrid fallback + + every consumer that gates on is_configured() depends on this.""" + client = SoulseekClient.__new__(SoulseekClient) + client.base_url = None + client.api_key = None + assert client.is_configured() is False + + +def test_is_configured_returns_true_when_base_url_set(): + """Pinning: configured client (slskd URL present) reports True.""" + client = SoulseekClient.__new__(SoulseekClient) + client.base_url = 'http://localhost:5030' + client.api_key = 'test-key' + assert client.is_configured() is True + + +# --------------------------------------------------------------------------- +# download() +# --------------------------------------------------------------------------- + + +@pytest.fixture +def configured_client(): + """A SoulseekClient with the slskd URL set but no real network. Tests + individually patch `_make_request` to return whatever shape they + want to exercise.""" + client = SoulseekClient.__new__(SoulseekClient) + client.base_url = 'http://localhost:5030' + client.api_key = 'test-key' + client.download_path = Path('./test_downloads') + return client + + +def test_download_returns_none_when_not_configured(): + """Pinning: an unconfigured client refuses downloads — returns + None silently rather than raising. Used as the soft-fail signal + by the orchestrator's per-source fallback chain.""" + client = SoulseekClient.__new__(SoulseekClient) + client.base_url = None + result = _run_async(client.download('user', 'song.flac', 1024)) + assert result is None + + +def test_download_hits_transfers_downloads_username_endpoint(configured_client): + """Pinning: the primary download endpoint is + `transfers/downloads/` POST. This shape was chosen to + match slskd's web-interface API exactly. Changing it breaks + every download against current slskd builds.""" + captured = [] + + async def fake_request(method, endpoint, json=None, **kwargs): + captured.append((method, endpoint, json)) + return {'id': 'dl-id-from-slskd'} + + with patch.object(configured_client, '_make_request', side_effect=fake_request): + result = _run_async(configured_client.download('user', 'song.flac', 1024)) + + assert result == 'dl-id-from-slskd' + method, endpoint, payload = captured[0] + assert method == 'POST' + assert endpoint == 'transfers/downloads/user' + # Payload is the slskd web-interface array format. + assert isinstance(payload, list) + assert payload[0]['filename'] == 'song.flac' + assert payload[0]['size'] == 1024 + + +def test_download_extracts_id_from_dict_response(configured_client): + """Pinning: when slskd returns `{id: ...}`, that's the + download_id the orchestrator uses to track the download.""" + with patch.object(configured_client, '_make_request', + AsyncMock(return_value={'id': 'abc123'})): + result = _run_async(configured_client.download('user', 'song.flac', 1024)) + assert result == 'abc123' + + +def test_download_extracts_id_from_list_response(configured_client): + """Pinning: slskd sometimes returns a list of file objects. + The first item's id is the download_id.""" + with patch.object(configured_client, '_make_request', + AsyncMock(return_value=[{'id': 'list-id'}, {'id': 'second'}])): + result = _run_async(configured_client.download('user', 'song.flac', 1024)) + assert result == 'list-id' + + +def test_download_falls_back_to_filename_when_no_id_in_response(configured_client): + """Pinning: defensive — older slskd builds returned 201 Created + with no id field. The client uses the filename as the download + identifier in that case so downstream tracking still works.""" + with patch.object(configured_client, '_make_request', + AsyncMock(return_value={'status': 'queued'})): + result = _run_async(configured_client.download('user', 'song.flac', 1024)) + assert result == 'song.flac' + + +# --------------------------------------------------------------------------- +# get_all_downloads() +# --------------------------------------------------------------------------- + + +def test_get_all_downloads_returns_empty_when_not_configured(): + client = SoulseekClient.__new__(SoulseekClient) + client.base_url = None + result = _run_async(client.get_all_downloads()) + assert result == [] + + +def test_get_all_downloads_parses_nested_user_directory_files_response(configured_client): + """Pinning: slskd's `transfers/downloads` returns + `[{username, directories: [{files: [...]}]}]`. The client + flattens that into a list of DownloadStatus objects, one per + file. Engine refactor's state aggregation depends on this shape.""" + fake_response = [ + { + 'username': 'peer1', + 'directories': [{ + 'files': [ + {'id': 'f1', 'filename': 'a.flac', 'state': 'InProgress', + 'size': 100, 'bytesTransferred': 50, 'averageSpeed': 1024}, + {'id': 'f2', 'filename': 'b.flac', 'state': 'Completed, Succeeded', + 'size': 200, 'bytesTransferred': 200, 'averageSpeed': 2048}, + ], + }], + }, + ] + + with patch.object(configured_client, '_make_request', + AsyncMock(return_value=fake_response)): + result = _run_async(configured_client.get_all_downloads()) + + assert len(result) == 2 + assert result[0].id == 'f1' + assert result[0].username == 'peer1' + assert result[0].state == 'InProgress' + assert result[1].id == 'f2' + # Pinning: 'Completed' state forces progress=100 regardless of source data. + assert result[1].progress == 100.0 + + +def test_get_all_downloads_endpoint_is_transfers_downloads(configured_client): + """Pinning: the listing endpoint is `transfers/downloads` (no + username). The 404'd `users/.../downloads` variant was tried + once and removed — keep it gone.""" + captured = [] + + async def fake_request(method, endpoint, **kwargs): + captured.append((method, endpoint)) + return [] + + with patch.object(configured_client, '_make_request', side_effect=fake_request): + _run_async(configured_client.get_all_downloads()) + + assert captured == [('GET', 'transfers/downloads')] + + +# --------------------------------------------------------------------------- +# cancel_download() +# --------------------------------------------------------------------------- + + +def test_cancel_download_returns_false_when_not_configured(): + client = SoulseekClient.__new__(SoulseekClient) + client.base_url = None + result = _run_async(client.cancel_download('dl-id', 'user', remove=False)) + assert result is False + + +def test_cancel_download_looks_up_username_when_not_provided(configured_client): + """Pinning: orchestrator may call cancel_download without a + username hint. The client falls back to scanning all downloads + to find which peer owns it. Engine refactor must preserve this + so existing API endpoints that don't pass username keep working.""" + fake_listing = [ + { + 'username': 'peer-owner', + 'directories': [{ + 'files': [{'id': 'target-dl', 'filename': 'x.flac', + 'state': 'InProgress', 'size': 0, + 'bytesTransferred': 0, 'averageSpeed': 0}], + }], + }, + ] + + captured_endpoints = [] + + async def fake_request(method, endpoint, **kwargs): + captured_endpoints.append((method, endpoint)) + if method == 'GET' and endpoint == 'transfers/downloads': + return fake_listing + # The DELETE for cancel — return success + return True + + with patch.object(configured_client, '_make_request', side_effect=fake_request): + _run_async(configured_client.cancel_download('target-dl', None, remove=False)) + + # The lookup hit get_all_downloads first, then the DELETE used the discovered username. + assert ('GET', 'transfers/downloads') in captured_endpoints + delete_calls = [(m, e) for m, e in captured_endpoints if m == 'DELETE'] + assert delete_calls, "Expected at least one DELETE after username lookup" + # The cancel endpoint URL contains the discovered username. + assert any('peer-owner' in e for _, e in delete_calls) + + +def test_cancel_download_returns_false_when_username_lookup_fails(configured_client): + """Pinning: if the download_id isn't in the active list, return + False rather than raising. Orchestrator treats False as "couldn't + cancel" and continues; an exception would propagate to the user.""" + with patch.object(configured_client, '_make_request', + AsyncMock(return_value=[])): + result = _run_async(configured_client.cancel_download('missing-id', None)) + assert result is False + + +# --------------------------------------------------------------------------- +# HTTP timeout config (issue #499 — prevent worker thread deadlock) +# --------------------------------------------------------------------------- + + +def test_default_timeout_constant_has_bounded_values(): + """Pin issue #499 fix: the module-level timeout config is defined + with a hard ceiling so an unresponsive slskd can't wedge the + download worker thread permanently. Any future change that drops + or unbounds the timeout would re-introduce the + 'downloads stop after 2-3 hours' deadlock.""" + from core.soulseek_client import _SLSKD_DEFAULT_TIMEOUT + import aiohttp + + assert isinstance(_SLSKD_DEFAULT_TIMEOUT, aiohttp.ClientTimeout) + # Total timeout must be set and bounded — prevents infinite hang. + assert _SLSKD_DEFAULT_TIMEOUT.total is not None + assert _SLSKD_DEFAULT_TIMEOUT.total > 0 + assert _SLSKD_DEFAULT_TIMEOUT.total <= 300, ( + f"Total timeout {_SLSKD_DEFAULT_TIMEOUT.total}s exceeds 5min " + "ceiling — slskd metadata calls should never legitimately take this long" + ) + # Connect timeout bounded — TCP connect to slskd should be fast. + assert _SLSKD_DEFAULT_TIMEOUT.connect is not None + assert _SLSKD_DEFAULT_TIMEOUT.connect <= 60 + + +def test_make_request_returns_none_on_timeout(configured_client): + """Pin: when the slskd HTTP call times out (asyncio.TimeoutError), + ``_make_request`` returns None rather than raising. The download + worker thread unblocks; the caller treats None as a normal failure + and the batch's stuck-detection later marks the task not_found. + Pre-fix this raised → propagated up the call stack → eventually + the worker thread died but only after wedging the executor pool.""" + + async def _raise_timeout(*args, **kwargs): + raise asyncio.TimeoutError("simulated slskd hang") + + # Patch aiohttp.ClientSession to return a session whose request() + # context manager raises TimeoutError on entry. + class _StubSession: + async def __aenter__(self): + return self + async def __aexit__(self, *args): + return None + + def request(self, *args, **kwargs): + class _Cm: + async def __aenter__(self_inner): + raise asyncio.TimeoutError("simulated slskd hang") + async def __aexit__(self_inner, *args): + return None + return _Cm() + + async def close(self): + return None + + with patch('aiohttp.ClientSession', return_value=_StubSession()): + result = _run_async(configured_client._make_request('GET', 'transfers/downloads')) + + assert result is None, "Timeout must return None, not raise" + + +def test_make_direct_request_returns_none_on_timeout(configured_client): + """Same pin for ``_make_direct_request`` (the non-/api/v0/ helper). + Both code paths are used by different slskd endpoints — neither + can be allowed to wedge.""" + + class _StubSession: + async def __aenter__(self): + return self + async def __aexit__(self, *args): + return None + + def request(self, *args, **kwargs): + class _Cm: + async def __aenter__(self_inner): + raise asyncio.TimeoutError("simulated slskd hang") + async def __aexit__(self_inner, *args): + return None + return _Cm() + + async def close(self): + return None + + with patch('aiohttp.ClientSession', return_value=_StubSession()): + result = _run_async(configured_client._make_direct_request('GET', 'health')) + + assert result is None diff --git a/tests/downloads/test_soundcloud_pinning.py b/tests/downloads/test_soundcloud_pinning.py new file mode 100644 index 00000000..4a5bc893 --- /dev/null +++ b/tests/downloads/test_soundcloud_pinning.py @@ -0,0 +1,131 @@ +"""Phase A pinning tests for SoundcloudClient — UPDATED for Phase C7. + +SoundCloud's quirk: 3-part filename `track_id||permalink_url||display_name` +because yt-dlp consumes the URL, not the track_id, to actually download. +The engine record holds both fields so the worker can call +`_download_sync(download_id, permalink_url, display_name)` correctly. +""" + +from __future__ import annotations + +import asyncio +import threading +from pathlib import Path +from unittest.mock import patch + +import pytest + +from core.download_engine import DownloadEngine +from core.soundcloud_client import SoundcloudClient + + +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture +def sc_client_with_engine(): + client = SoundcloudClient.__new__(SoundcloudClient) + client.download_path = Path('./test_sc_downloads') + client.shutdown_check = None + client._engine = None + engine = DownloadEngine() + client.set_engine(engine) + return client, engine + + +def test_download_returns_none_for_filename_with_too_few_parts(sc_client_with_engine): + client, _ = sc_client_with_engine + result = _run_async(client.download('soundcloud', 'just-id-no-url', 0)) + assert result is None + + +def test_download_returns_none_for_empty_track_id_or_url(sc_client_with_engine): + client, _ = sc_client_with_engine + assert _run_async(client.download('soundcloud', '||https://x.com/y', 0)) is None + assert _run_async(client.download('soundcloud', 'track123||', 0)) is None + + +def test_download_raises_when_engine_not_wired(): + """Defensive: client without engine reference must raise so the + orchestrator's download_with_fallback surfaces the error and + moves on to the next source. Returning None silently would drop + the download with no user feedback (per JohnBaumb).""" + import pytest + client = SoundcloudClient.__new__(SoundcloudClient) + client._engine = None + with pytest.raises(RuntimeError, match="engine reference"): + _run_async(client.download('soundcloud', 'v||t', 0)) + + +def test_download_accepts_three_part_filename_with_display(sc_client_with_engine): + """Pinning: 3-part filename `track_id||permalink_url||display` + is the canonical form. All three fields go into the engine record.""" + client, engine = sc_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/done.mp3' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download( + 'soundcloud', + 'sc-12345||https://soundcloud.com/artist/song||Some Display Title', + 0, + )) + started.wait(timeout=1.0) + record = engine.get_record('soundcloud', download_id) + + assert record['track_id'] == 'sc-12345' + assert record['permalink_url'] == 'https://soundcloud.com/artist/song' + assert record['display_name'] == 'Some Display Title' + release.set() + + +def test_download_falls_back_display_name_to_track_id_when_two_part(sc_client_with_engine): + """Pinning: 2-part filename → display name = track_id.""" + client, engine = sc_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/x.mp3' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download( + 'soundcloud', 'sc-99||https://soundcloud.com/x/y', 0, + )) + started.wait(timeout=1.0) + assert engine.get_record('soundcloud', download_id)['display_name'] == 'sc-99' + release.set() + + +def test_download_populates_engine_record_with_initial_state(sc_client_with_engine): + client, engine = sc_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/x.mp3' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download( + 'soundcloud', 'sc-1||https://soundcloud.com/x||Title', 0, + )) + started.wait(timeout=1.0) + record = engine.get_record('soundcloud', download_id) + assert record['username'] == 'soundcloud' + assert record['state'] in ('Initializing', 'InProgress, Downloading') + assert 'permalink_url' in record + release.set() diff --git a/tests/downloads/test_tidal_pinning.py b/tests/downloads/test_tidal_pinning.py new file mode 100644 index 00000000..74327f37 --- /dev/null +++ b/tests/downloads/test_tidal_pinning.py @@ -0,0 +1,172 @@ +"""Phase A pinning tests for TidalDownloadClient — UPDATED for Phase C3. + +Post-C3 the client no longer owns its own ``active_downloads`` dict +or thread spawn — both moved into the engine's BackgroundDownloadWorker. +Pinning tests now read state from ``engine.get_record('tidal', ...)`` +instead of ``client.active_downloads[...]``. +""" + +from __future__ import annotations + +import asyncio +import threading +from pathlib import Path +from unittest.mock import patch + +import pytest + +from core.download_engine import DownloadEngine +from core.tidal_download_client import TidalDownloadClient + + +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture +def tidal_client_with_engine(): + client = TidalDownloadClient.__new__(TidalDownloadClient) + client.download_path = Path('./test_tidal_downloads') + client.shutdown_check = None + client.session = None + client._device_auth_future = None + client._device_auth_link = None + client._engine = None + + engine = DownloadEngine() + client.set_engine(engine) + return client, engine + + +# --------------------------------------------------------------------------- +# is_configured / is_authenticated +# --------------------------------------------------------------------------- + + +def test_is_authenticated_false_when_no_session(tidal_client_with_engine): + client, _ = tidal_client_with_engine + assert client.is_authenticated() is False + + +def test_is_authenticated_false_when_session_check_login_raises(tidal_client_with_engine): + client, _ = tidal_client_with_engine + fake_session = type('FakeSession', (), { + 'check_login': lambda self: (_ for _ in ()).throw(RuntimeError("expired")), + })() + client.session = fake_session + assert client.is_authenticated() is False + + +# --------------------------------------------------------------------------- +# download() — filename parsing + id contract +# --------------------------------------------------------------------------- + + +def test_download_returns_none_for_invalid_filename_format(tidal_client_with_engine): + client, _ = tidal_client_with_engine + result = _run_async(client.download('tidal', 'no-separator', 0)) + assert result is None + + +def test_download_returns_none_for_non_integer_track_id(tidal_client_with_engine): + client, _ = tidal_client_with_engine + result = _run_async(client.download('tidal', 'not-int||title', 0)) + assert result is None + + +def test_download_raises_when_engine_not_wired(): + """Defensive: client without engine reference must raise so the + orchestrator's download_with_fallback surfaces the error and + moves on to the next source. Returning None silently would drop + the download with no user feedback (per JohnBaumb).""" + import pytest + client = TidalDownloadClient.__new__(TidalDownloadClient) + client._engine = None + with pytest.raises(RuntimeError, match="engine reference"): + _run_async(client.download('tidal', 'v||t', 0)) + + +def test_download_returns_uuid_for_valid_filename(tidal_client_with_engine): + client, _ = tidal_client_with_engine + with patch.object(client, '_download_sync', return_value='/tmp/x.flac'): + result = _run_async(client.download('tidal', '12345||Some Song', 0)) + assert result is not None + assert len(result) == 36 + + +def test_download_populates_engine_record_with_initial_state(tidal_client_with_engine): + client, engine = tidal_client_with_engine + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/done.flac' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async(client.download('tidal', '999||My Tidal Song', 0)) + started.wait(timeout=1.0) + record = engine.get_record('tidal', download_id) + + assert record is not None + assert record['id'] == download_id + assert record['filename'] == '999||My Tidal Song' + assert record['username'] == 'tidal' + assert record['state'] in ('Initializing', 'InProgress, Downloading') + assert record['progress'] == 0.0 + assert record['track_id'] == 999 # parsed as int + assert record['display_name'] == 'My Tidal Song' + assert record['file_path'] is None + release.set() + + +# --------------------------------------------------------------------------- +# Query / cancel — engine-backed reads +# --------------------------------------------------------------------------- + + +def test_get_all_downloads_reads_engine_records(tidal_client_with_engine): + client, engine = tidal_client_with_engine + engine.add_record('tidal', 'dl-1', { + 'id': 'dl-1', 'filename': '111||Song A', 'username': 'tidal', + 'state': 'InProgress, Downloading', 'progress': 50.0, + 'size': 1000, 'transferred': 500, 'speed': 100, + }) + engine.add_record('tidal', 'dl-2', { + 'id': 'dl-2', 'filename': '222||Song B', 'username': 'tidal', + 'state': 'Completed, Succeeded', 'progress': 100.0, + 'size': 2000, 'transferred': 2000, 'speed': 0, + }) + result = _run_async(client.get_all_downloads()) + assert len(result) == 2 + assert {r.id for r in result} == {'dl-1', 'dl-2'} + assert {r.username for r in result} == {'tidal'} + + +def test_cancel_download_marks_cancelled(tidal_client_with_engine): + client, engine = tidal_client_with_engine + engine.add_record('tidal', 'dl-1', {'id': 'dl-1', 'state': 'InProgress, Downloading'}) + + ok = _run_async(client.cancel_download('dl-1', None, remove=False)) + assert ok is True + assert engine.get_record('tidal', 'dl-1')['state'] == 'Cancelled' + + ok = _run_async(client.cancel_download('dl-1', None, remove=True)) + assert ok is True + assert engine.get_record('tidal', 'dl-1') is None + + +def test_clear_all_completed_drops_only_terminal_records(tidal_client_with_engine): + client, engine = tidal_client_with_engine + engine.add_record('tidal', 'done', {'id': 'done', 'state': 'Completed, Succeeded'}) + engine.add_record('tidal', 'live', {'id': 'live', 'state': 'InProgress, Downloading'}) + + _run_async(client.clear_all_completed_downloads()) + + assert engine.get_record('tidal', 'done') is None + assert engine.get_record('tidal', 'live') is not None diff --git a/tests/downloads/test_youtube_pinning.py b/tests/downloads/test_youtube_pinning.py new file mode 100644 index 00000000..4637af97 --- /dev/null +++ b/tests/downloads/test_youtube_pinning.py @@ -0,0 +1,212 @@ +"""Phase A pinning tests for YouTubeClient — UPDATED for Phase C2. + +Post-C2 the client no longer owns its own ``active_downloads`` dict +or thread spawn — both moved into the engine's BackgroundDownloadWorker. +These tests still pin the same OBSERVABLE CONTRACT (filename +encoding, UUID download_id, initial-record schema, source-specific +extras like video_id/url/title) but read state from +``engine.get_record(...)`` instead of ``client.active_downloads[...]``. + +What pre-C2 pinning tests caught and what these still catch: +- Filename format: `video_id||title` ✓ +- Invalid filename → None ✓ +- UUID download_id format ✓ +- Per-download record schema (id, filename, username, state, + progress, video_id, url, title, file_path) ✓ +- Source name in record's username slot is `'youtube'` ✓ + +What dropped (covered by other tests): +- Direct thread-spawn assertion (engine.worker has its own tests). +- `_download_thread_worker` target (gone from client; engine owns + it). +""" + +from __future__ import annotations + +import asyncio +import threading +from pathlib import Path +from unittest.mock import patch + +import pytest + +from core.download_engine import DownloadEngine +from core.youtube_client import YouTubeClient + + +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture +def yt_client_with_engine(): + """A bare YouTubeClient wired into a real engine. The engine + callback is invoked manually since we bypass orchestrator init.""" + client = YouTubeClient.__new__(YouTubeClient) + client.download_path = Path('./test_yt_downloads') + client.shutdown_check = None + client.matching_engine = None + client._download_delay = 3 + client.current_download_id = None + client.current_download_progress = { + 'status': 'idle', 'percent': 0.0, 'downloaded_bytes': 0, + 'total_bytes': 0, 'speed': 0, 'eta': 0, 'filename': '', + } + client.progress_callback = None + client.download_opts = {} + client._engine = None + + engine = DownloadEngine() + client.set_engine(engine) + return client, engine + + +# --------------------------------------------------------------------------- +# download() — filename parsing + id contract +# --------------------------------------------------------------------------- + + +def test_download_returns_none_for_invalid_filename_format(yt_client_with_engine): + """Pinning: missing `||` → None (not exception).""" + client, _ = yt_client_with_engine + result = _run_async(client.download('youtube', 'no-separator', 0)) + assert result is None + + +def test_download_raises_when_engine_not_wired(): + """Defensive: client without engine reference must raise so the + orchestrator's download_with_fallback surfaces the error and + moves on to the next source. Returning None silently would drop + the download with no user feedback (per JohnBaumb).""" + import pytest + client = YouTubeClient.__new__(YouTubeClient) + client._engine = None + with pytest.raises(RuntimeError, match="engine reference"): + _run_async(client.download('youtube', 'v||t', 0)) + + +def test_download_returns_uuid_download_id_for_valid_filename(yt_client_with_engine): + """Pinning: valid `video_id||title` → UUID download_id.""" + client, engine = yt_client_with_engine + + # Patch _download_sync so the worker thread's impl returns + # without doing real yt-dlp work. + with patch.object(client, '_download_sync', return_value='/tmp/x.mp3'): + result = _run_async(client.download('youtube', 'abc123||Some Song', 0)) + + assert result is not None + assert len(result) == 36 + assert result.count('-') == 4 + + +def test_download_populates_engine_record_with_initial_state(yt_client_with_engine): + """Pinning: per-download record schema. STATE LOCATION CHANGED + in C2 (now in engine), but the SHAPE of the record is the same + — frontend / status APIs / context-key matching depend on these + keys.""" + client, engine = yt_client_with_engine + + # Hold the impl so we can read 'Initializing' / 'InProgress' state + # before the worker completes. + started = threading.Event() + release = threading.Event() + + def slow_impl(*args, **kwargs): + started.set() + release.wait(timeout=1.0) + return '/tmp/done.mp3' + + with patch.object(client, '_download_sync', side_effect=slow_impl): + download_id = _run_async( + client.download('youtube', 'video123||My Title', 5000) + ) + started.wait(timeout=1.0) + record = engine.get_record('youtube', download_id) + + assert record is not None + assert record['id'] == download_id + assert record['filename'] == 'video123||My Title' # ORIGINAL form + assert record['username'] == 'youtube' + assert record['state'] in ('Initializing', 'InProgress, Downloading') + assert record['progress'] == 0.0 + assert record['file_path'] is None + # Source-specific extras must merge into the record. + assert record['video_id'] == 'video123' + assert record['url'] == 'https://www.youtube.com/watch?v=video123' + assert record['title'] == 'My Title' + + release.set() + + +def test_set_engine_configures_worker_delay(yt_client_with_engine): + """Pinning: when engine is wired, the YouTube download_delay + config (3s default) propagates to the worker so successive + downloads serialize with the same gap they did pre-C2.""" + client, engine = yt_client_with_engine + # Default delay is 3s. + assert engine.worker._get_delay('youtube') == 3.0 + + +def test_rate_limit_policy_reflects_configured_delay(yt_client_with_engine): + """Pinning (Phase E): YouTube's rate_limit_policy() returns a + RateLimitPolicy with the configured download_delay (3s default + from `youtube.download_delay` config). Engine reads this at + register_plugin time.""" + client, _ = yt_client_with_engine + policy = client.rate_limit_policy() + assert policy.download_delay_seconds == 3.0 + assert policy.download_concurrency == 1 + + +# --------------------------------------------------------------------------- +# Query / cancel — engine-backed reads +# --------------------------------------------------------------------------- + + +def test_get_all_downloads_reads_engine_records(yt_client_with_engine): + client, engine = yt_client_with_engine + + # Seed engine with a fake record to mirror what dispatch would do. + engine.add_record('youtube', 'dl-1', { + 'id': 'dl-1', 'filename': 'v||t', 'username': 'youtube', + 'state': 'InProgress, Downloading', 'progress': 50.0, + 'size': 1000, 'transferred': 500, 'speed': 100, + }) + result = _run_async(client.get_all_downloads()) + assert len(result) == 1 + assert result[0].id == 'dl-1' + assert result[0].state == 'InProgress, Downloading' + + +def test_cancel_download_marks_cancelled_and_optionally_removes(yt_client_with_engine): + client, engine = yt_client_with_engine + + engine.add_record('youtube', 'dl-1', { + 'id': 'dl-1', 'filename': 'v||t', 'username': 'youtube', + 'state': 'InProgress, Downloading', 'progress': 50.0, + }) + + ok = _run_async(client.cancel_download('dl-1', None, remove=False)) + assert ok is True + assert engine.get_record('youtube', 'dl-1')['state'] == 'Cancelled' + + ok = _run_async(client.cancel_download('dl-1', None, remove=True)) + assert ok is True + assert engine.get_record('youtube', 'dl-1') is None + + +def test_clear_all_completed_drops_only_terminal_records(yt_client_with_engine): + client, engine = yt_client_with_engine + engine.add_record('youtube', 'done', {'id': 'done', 'state': 'Completed, Succeeded'}) + engine.add_record('youtube', 'erred', {'id': 'erred', 'state': 'Errored'}) + engine.add_record('youtube', 'live', {'id': 'live', 'state': 'InProgress, Downloading'}) + + _run_async(client.clear_all_completed_downloads()) + + assert engine.get_record('youtube', 'done') is None + assert engine.get_record('youtube', 'erred') is None + assert engine.get_record('youtube', 'live') is not None diff --git a/tests/enrichment/__init__.py b/tests/enrichment/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/enrichment/test_manual_match_honoring.py b/tests/enrichment/test_manual_match_honoring.py new file mode 100644 index 00000000..c6031461 --- /dev/null +++ b/tests/enrichment/test_manual_match_honoring.py @@ -0,0 +1,302 @@ +"""Tests for ``core.enrichment.manual_match_honoring.honor_stored_match``. + +The helper is the shared "fast-path enrichment via stored source ID" +used by every per-source enrichment worker (Spotify / iTunes / Deezer +/ Discogs / MusicBrainz / AudioDB / Tidal / Qobuz). It reads the +stored ID from a configurable column, fetches via a caller-supplied +client method, and invokes a caller-supplied update callback. Pin +the contract so per-worker wiring can rely on uniform semantics. + +Issue #501: enrichment workers were running fuzzy name search and +overwriting manually-set source IDs. This helper is the lift point — +all 8 workers will plug in the same way. +""" + +from __future__ import annotations + +import sqlite3 +from unittest.mock import MagicMock + +import pytest + +from core.enrichment.manual_match_honoring import honor_stored_match + + +# --------------------------------------------------------------------------- +# Fake DB fixture (just enough to exercise _read_id_column) +# --------------------------------------------------------------------------- + + +class _FakeConn: + """Minimal sqlite3.Connection-like that the helper can use.""" + + def __init__(self, real_conn): + self._real = real_conn + + def cursor(self): + return self._real.cursor() + + def close(self): + # Don't actually close — tests share the connection. + pass + + +class _FakeDB: + """Stand-in MusicDatabase. Supports ``_get_connection()`` returning + a wrapper that doesn't close, so per-test in-memory state survives + across helper invocations.""" + + def __init__(self): + self._conn = sqlite3.connect(":memory:") + self._conn.row_factory = sqlite3.Row + cur = self._conn.cursor() + cur.execute(""" + CREATE TABLE albums ( + id INTEGER PRIMARY KEY, + spotify_album_id TEXT, + deezer_id TEXT, + itunes_album_id TEXT + ) + """) + cur.execute(""" + CREATE TABLE tracks ( + id INTEGER PRIMARY KEY, + spotify_track_id TEXT, + deezer_id TEXT + ) + """) + self._conn.commit() + + def insert_album(self, album_id, **id_columns): + cols = ['id'] + list(id_columns.keys()) + placeholders = ','.join('?' for _ in cols) + values = [album_id] + list(id_columns.values()) + self._conn.execute( + f"INSERT INTO albums ({','.join(cols)}) VALUES ({placeholders})", + values, + ) + self._conn.commit() + + def insert_track(self, track_id, **id_columns): + cols = ['id'] + list(id_columns.keys()) + placeholders = ','.join('?' for _ in cols) + values = [track_id] + list(id_columns.values()) + self._conn.execute( + f"INSERT INTO tracks ({','.join(cols)}) VALUES ({placeholders})", + values, + ) + self._conn.commit() + + def _get_connection(self): + return _FakeConn(self._conn) + + +@pytest.fixture +def db(): + return _FakeDB() + + +# --------------------------------------------------------------------------- +# Stored-ID fast path (the new behavior) +# --------------------------------------------------------------------------- + + +def test_honors_stored_id_when_present(db): + """Pin: stored ID found → fetch → on_match called → returns True. + Caller skips its search-by-name flow.""" + db.insert_album(42, spotify_album_id='SP-ABC') + api_payload = {'id': 'SP-ABC', 'name': 'Real Album'} + fetch = MagicMock(return_value=api_payload) + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='albums', entity_id=42, + id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + log_prefix='Spotify', + ) + + assert result is True + fetch.assert_called_once_with('SP-ABC') + on_match.assert_called_once_with(42, 'SP-ABC', api_payload) + + +def test_returns_false_when_no_stored_id(db): + """Pin: no stored ID → returns False, no fetch attempted, no + callback. Caller proceeds with its search-by-name fallback.""" + db.insert_album(42, spotify_album_id=None) + fetch = MagicMock() + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='albums', entity_id=42, + id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + log_prefix='Spotify', + ) + + assert result is False + fetch.assert_not_called() + on_match.assert_not_called() + + +def test_returns_false_when_stored_id_empty_string(db): + """Pin: empty string treated same as NULL — no fetch, return False.""" + db.insert_album(42, spotify_album_id='') + fetch = MagicMock() + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='albums', entity_id=42, + id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + assert result is False + fetch.assert_not_called() + + +def test_returns_false_when_entity_not_in_db(db): + """Pin: missing row → returns False, no fetch, no callback.""" + fetch = MagicMock() + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='albums', entity_id=999, + id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + assert result is False + fetch.assert_not_called() + + +# --------------------------------------------------------------------------- +# Failure paths — fall through to search instead of crashing the worker +# --------------------------------------------------------------------------- + + +def test_returns_false_when_fetch_raises(db): + """Pin: client.get_X(stored_id) raises → caught, logged at warning, + returns False so caller falls through to search. Worker must not + crash on a transient API failure.""" + db.insert_album(42, spotify_album_id='SP-ABC') + fetch = MagicMock(side_effect=RuntimeError("API down")) + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='albums', entity_id=42, + id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + assert result is False + on_match.assert_not_called() + + +def test_returns_false_when_fetch_returns_none(db): + """Pin: stored ID points at a removed/invalid catalog entry → + fetch returns None → falls through to search instead of writing + junk to DB.""" + db.insert_album(42, spotify_album_id='SP-DEAD') + fetch = MagicMock(return_value=None) + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='albums', entity_id=42, + id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + assert result is False + on_match.assert_not_called() + + +def test_returns_false_when_fetch_returns_empty_dict(db): + """Pin: empty dict treated same as None — falsy result skips + callback.""" + db.insert_album(42, spotify_album_id='SP-EMPTY') + fetch = MagicMock(return_value={}) + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='albums', entity_id=42, + id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + assert result is False + on_match.assert_not_called() + + +def test_on_match_exceptions_propagate(db): + """Pin: exceptions inside on_match (DB write errors) propagate to + the worker — they're real errors the worker should surface, not + swallowed silently.""" + db.insert_album(42, spotify_album_id='SP-ABC') + fetch = MagicMock(return_value={'id': 'SP-ABC'}) + on_match = MagicMock(side_effect=ValueError("bad write")) + + with pytest.raises(ValueError, match="bad write"): + honor_stored_match( + db=db, entity_table='albums', entity_id=42, + id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + +# --------------------------------------------------------------------------- +# Per-table / per-column wiring +# --------------------------------------------------------------------------- + + +def test_works_with_tracks_table(db): + """Pin: ``entity_table='tracks'`` works the same as 'albums' — the + helper is generic across both.""" + db.insert_track(7, spotify_track_id='SP-T-1') + fetch = MagicMock(return_value={'id': 'SP-T-1'}) + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='tracks', entity_id=7, + id_column='spotify_track_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + assert result is True + fetch.assert_called_once_with('SP-T-1') + + +def test_works_with_alternate_columns(db): + """Pin: ``id_column`` is configurable so each worker reads its own + column (deezer_id, itunes_album_id, etc).""" + db.insert_album(42, deezer_id='12345', itunes_album_id='IT-99') + fetch = MagicMock(return_value={'id': '12345'}) + on_match = MagicMock() + + # Read the deezer_id column even though spotify_album_id exists too. + result = honor_stored_match( + db=db, entity_table='albums', entity_id=42, + id_column='deezer_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + assert result is True + fetch.assert_called_once_with('12345') + + +def test_rejects_invalid_table_name(db): + """Pin: defensive — only known tables (albums/tracks/artists) + accepted. Avoids SQL injection via crafted table name even though + every caller is hard-coded.""" + fetch = MagicMock() + on_match = MagicMock() + + result = honor_stored_match( + db=db, entity_table='not_a_real_table; DROP TABLE albums', + entity_id=1, id_column='spotify_album_id', + client_fetch_fn=fetch, on_match_fn=on_match, + ) + + assert result is False + fetch.assert_not_called() diff --git a/tests/imports/test_auto_import_live_progress.py b/tests/imports/test_auto_import_live_progress.py new file mode 100644 index 00000000..584387b3 --- /dev/null +++ b/tests/imports/test_auto_import_live_progress.py @@ -0,0 +1,350 @@ +"""Regression tests for auto-import live-progress visibility. + +Reported case: dropping an album into the staging folder, the +import processes track-by-track but the UI shows nothing in the +auto-import history list and the status indicator stays at +"Watching" — no live progress visible. After a multi-minute +processing window the row finally appears with status='completed', +but for the duration of the import the user has no signal that +anything is happening. + +Two pre-existing gaps caused this: + +1. ``_record_result`` only fires AFTER ``_process_matches`` returns. + For a 14-track album with ~30s/track post-processing, that's a + 7-minute window with no DB row → nothing for the UI's + ``/api/auto-import/results`` to return. + +2. ``_current_status`` only ever transitioned between 'idle' and + 'scanning' — never 'processing'. ``get_status()`` had no per- + track index/name fields, so the UI had no way to render + "Processing track 3/14: Mine". + +These tests pin both fixes: +- An in-progress ``auto_import_history`` row gets inserted up-front + and updated to the final status when processing completes. +- ``get_status()`` exposes ``current_status='processing'`` plus + per-track index / total / name during the per-track loop. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Any, Dict, List +from unittest.mock import MagicMock + +import pytest + + +# --------------------------------------------------------------------------- +# Stubs + fixtures +# --------------------------------------------------------------------------- + + +@dataclass +class _FakeCandidate: + path: str + name: str + audio_files: List[str] = field(default_factory=list) + disc_structure: Dict[int, List[str]] = field(default_factory=dict) + folder_hash: str = "fake-hash" + is_single: bool = False + + +@pytest.fixture(autouse=True) +def _stub_metadata_clients(monkeypatch): + """Avoid real HTTP calls during context construction.""" + try: + from core.imports import album as album_mod + monkeypatch.setattr(album_mod, "get_client_for_source", lambda _src: None) + except Exception: + pass + yield + + +@pytest.fixture +def auto_import_worker(tmp_path): + """AutoImportWorker with a no-op process_callback that captures + per-track state at call time so we can verify the live-progress + fields advance during the loop.""" + from core.auto_import_worker import AutoImportWorker + + captured = [] + + fake_db = MagicMock() + fake_cfg = MagicMock() + fake_cfg.get.side_effect = lambda key, default=None: default + + worker = AutoImportWorker( + database=fake_db, + staging_path=str(tmp_path), + transfer_path=str(tmp_path / 'transfer'), + process_callback=None, # set below + config_manager=fake_cfg, + automation_engine=None, + ) + + def _capturing_callback(key, ctx, path): + # Snapshot the live-progress state AT THE MOMENT the callback + # fires for a track. The UI polls get_status() at the same + # cadence so this is what an interleaved poll would see. + captured.append({ + 'key': key, + 'current_status': worker._current_status, + 'current_folder': worker._current_folder, + 'current_track_index': worker._current_track_index, + 'current_track_total': worker._current_track_total, + 'current_track_name': worker._current_track_name, + }) + + worker._process_callback = _capturing_callback + worker._captured = captured + return worker + + +def _make_match_result(track_count: int = 1) -> Dict[str, Any]: + return { + 'album_data': { + 'id': 'album-1', 'total_tracks': track_count, + 'album_type': 'album', 'release_date': '2024-01-01', + 'images': [{'url': 'https://img.example/cover.jpg'}], + 'artists': [{'name': 'A', 'id': 'artist-1'}], + }, + 'total_tracks': track_count, + 'matched_count': track_count, + 'confidence': 0.95, + } + + +def _make_identification(**overrides) -> Dict[str, Any]: + base = { + 'source': 'deezer', + 'artist_name': 'A', + 'artist_id': 'artist-1', + 'album_name': 'Test Album', + 'album_id': 'album-1', + 'image_url': 'https://img.example/cover.jpg', + 'release_date': '2024-01-01', + 'method': 'tags', + } + base.update(overrides) + return base + + +# --------------------------------------------------------------------------- +# get_status surfaces new fields +# --------------------------------------------------------------------------- + + +class TestGetStatusExposesLiveProgressFields: + def test_initial_state_has_zero_track_progress(self, auto_import_worker): + status = auto_import_worker.get_status() + assert status['current_track_index'] == 0 + assert status['current_track_total'] == 0 + assert status['current_track_name'] == '' + assert status['current_status'] == 'idle' + + +# --------------------------------------------------------------------------- +# Per-track progress advances during _process_matches +# --------------------------------------------------------------------------- + + +class TestPerTrackProgressUpdates: + def test_track_index_advances_during_loop(self, auto_import_worker, tmp_path): + """As _process_matches iterates tracks, the live-progress fields + must reflect the current index so a polling UI sees 1/3, 2/3, + 3/3 instead of nothing.""" + files = [] + for n in range(1, 4): + f = tmp_path / f"0{n}.mp3" + f.write_bytes(b"fake") + files.append(f) + + candidate = _FakeCandidate(path=str(tmp_path), name="Album") + identification = _make_identification() + match_result = _make_match_result(track_count=3) + match_result['matches'] = [ + {'track': {'id': f't{n}', 'name': f'Track {n}', + 'track_number': n, 'disc_number': 1, + 'duration_ms': 200000, 'artists': [{'name': 'A'}]}, + 'file': str(files[n - 1]), 'confidence': 0.95} + for n in range(1, 4) + ] + + auto_import_worker._process_matches(candidate, identification, match_result) + + captured = auto_import_worker._captured + assert len(captured) == 3 + assert [c['current_track_index'] for c in captured] == [1, 2, 3] + assert all(c['current_track_total'] == 3 for c in captured) + assert [c['current_track_name'] for c in captured] == ['Track 1', 'Track 2', 'Track 3'] + + def test_track_total_set_before_first_callback(self, auto_import_worker, tmp_path): + """The denominator must be in place when the FIRST track's + callback fires — otherwise the UI's first poll would render + '1/0' nonsense.""" + f = tmp_path / "01.mp3" + f.write_bytes(b"fake") + candidate = _FakeCandidate(path=str(tmp_path), name="Album") + identification = _make_identification() + match_result = _make_match_result(track_count=5) + match_result['matches'] = [ + {'track': {'id': 't1', 'name': 'Only Track', + 'track_number': 1, 'disc_number': 1, + 'duration_ms': 200000, 'artists': [{'name': 'A'}]}, + 'file': str(f), 'confidence': 0.95} + ] + + auto_import_worker._process_matches(candidate, identification, match_result) + + # Only one callback fired but track_total reflects the full + # match_result.matches length the loop opened with. + assert auto_import_worker._captured[0]['current_track_total'] == 1 + + +# --------------------------------------------------------------------------- +# In-progress + finalize DB round trip +# --------------------------------------------------------------------------- + + +class _RealishDB: + """Tiny in-memory SQLite shim that mimics MusicDatabase's + _get_connection() with a realistic auto_import_history schema.""" + + def __init__(self): + import sqlite3 + self._conn = sqlite3.connect(':memory:') + self._conn.row_factory = sqlite3.Row + self._conn.execute(""" + CREATE TABLE auto_import_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + folder_name TEXT, folder_path TEXT, folder_hash TEXT, + status TEXT, confidence REAL, album_id TEXT, album_name TEXT, + artist_name TEXT, image_url TEXT, total_files INTEGER, + matched_files INTEGER, match_data TEXT, + identification_method TEXT, error_message TEXT, + processed_at TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + self._conn.commit() + + def _get_connection(self): + # Wrap so .close() doesn't drop our in-memory DB. + class _NoCloseConn: + def __init__(_s, real): _s._real = real + def __getattr__(_s, name): return getattr(_s._real, name) + def close(_s): pass + return _NoCloseConn(self._conn) + + +@pytest.fixture +def real_db_worker(tmp_path): + from core.auto_import_worker import AutoImportWorker + db = _RealishDB() + fake_cfg = MagicMock() + fake_cfg.get.side_effect = lambda key, default=None: default + worker = AutoImportWorker( + database=db, staging_path=str(tmp_path), transfer_path=str(tmp_path), + process_callback=lambda *a, **k: None, + config_manager=fake_cfg, automation_engine=None, + ) + worker._db = db # for direct DB inspection + return worker + + +class TestInProgressRowLifecycle: + def test_record_in_progress_inserts_row_with_processing_status(self, real_db_worker): + candidate = _FakeCandidate(path="/x", name="Album", audio_files=['a.mp3']) + match_result = _make_match_result(track_count=3) + identification = _make_identification() + + row_id = real_db_worker._record_in_progress(candidate, identification, match_result) + assert row_id is not None and row_id > 0 + + cur = real_db_worker._db._get_connection().cursor() + cur.execute("SELECT status, album_name, artist_name, processed_at FROM auto_import_history WHERE id = ?", + (row_id,)) + row = cur.fetchone() + assert row['status'] == 'processing' + assert row['album_name'] == 'Test Album' + assert row['artist_name'] == 'A' + assert row['processed_at'] is None # Not finalized yet + + def test_finalize_result_updates_existing_row(self, real_db_worker): + candidate = _FakeCandidate(path="/x", name="Album", audio_files=['a.mp3']) + match_result = _make_match_result(track_count=3) + identification = _make_identification() + + row_id = real_db_worker._record_in_progress(candidate, identification, match_result) + real_db_worker._finalize_result(row_id, 'completed', 0.97) + + cur = real_db_worker._db._get_connection().cursor() + cur.execute("SELECT status, confidence, processed_at FROM auto_import_history WHERE id = ?", + (row_id,)) + row = cur.fetchone() + assert row['status'] == 'completed' + assert row['confidence'] == 0.97 + assert row['processed_at'] is not None # Set on completed status + + # Same row, not a second insert + cur.execute("SELECT COUNT(*) FROM auto_import_history") + assert cur.fetchone()[0] == 1 + + def test_finalize_result_failed_status_clears_processed_at(self, real_db_worker): + candidate = _FakeCandidate(path="/x", name="Album", audio_files=['a.mp3']) + match_result = _make_match_result(track_count=3) + row_id = real_db_worker._record_in_progress(candidate, _make_identification(), match_result) + + real_db_worker._finalize_result(row_id, 'failed', 0.0, error_message='something broke') + + cur = real_db_worker._db._get_connection().cursor() + cur.execute("SELECT status, error_message, processed_at FROM auto_import_history WHERE id = ?", + (row_id,)) + row = cur.fetchone() + assert row['status'] == 'failed' + assert row['error_message'] == 'something broke' + assert row['processed_at'] is None # Only set on completed + + def test_finalize_with_none_row_id_is_noop(self, real_db_worker): + """If _record_in_progress failed (DB error, etc.), it returns + None. Finalize must be safe to call with None and not crash.""" + real_db_worker._finalize_result(None, 'completed', 1.0) + # No assertion needed — just verifying no exception + cur = real_db_worker._db._get_connection().cursor() + cur.execute("SELECT COUNT(*) FROM auto_import_history") + assert cur.fetchone()[0] == 0 + + +# --------------------------------------------------------------------------- +# Live progress reset on completion +# --------------------------------------------------------------------------- + + +class TestLiveProgressResets: + def test_progress_fields_cleared_after_loop(self, auto_import_worker, tmp_path): + """Once _process_matches returns, the per-track fields go back + to zero — otherwise the UI would show stale 'processing 14/14: + last track' forever.""" + f = tmp_path / "t.mp3" + f.write_bytes(b"fake") + candidate = _FakeCandidate(path=str(tmp_path), name="Album") + identification = _make_identification() + match_result = _make_match_result(track_count=1) + match_result['matches'] = [{ + 'track': {'id': 't1', 'name': 'T', 'track_number': 1, 'disc_number': 1, + 'duration_ms': 200000, 'artists': [{'name': 'A'}]}, + 'file': str(f), 'confidence': 0.95, + }] + + auto_import_worker._process_matches(candidate, identification, match_result) + + # During the loop, progress was set (verified by capture). + assert auto_import_worker._captured[0]['current_track_index'] == 1 + # After the loop, _process_matches itself doesn't reset — the + # outer _scan_cycle does. But the captured snapshot mid-loop + # should at least show non-zero values, proving the mechanism + # works. + assert auto_import_worker._captured[0]['current_track_total'] == 1 diff --git a/tests/imports/test_build_single_import_context_typed_path.py b/tests/imports/test_build_single_import_context_typed_path.py new file mode 100644 index 00000000..8400f557 --- /dev/null +++ b/tests/imports/test_build_single_import_context_typed_path.py @@ -0,0 +1,131 @@ +"""Pin the typed-path migration of `_build_single_import_context_payload`. + +Same pattern as the `_build_album_info` migration: when the caller +passes a known source, the embedded album blob inside the track +response is dispatched through `Album.from__dict()` and the +resulting typed Album drives the album_payload. Legacy duck-typed +extraction stays as the fallback. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from core.imports import resolution + + +SAMPLE_SPOTIFY_TRACK = { + 'id': 'tr1', + 'name': 'HUMBLE.', + 'artists': [{'id': 'kdot', 'name': 'Kendrick Lamar'}], + 'album': { + 'id': 'sp_album', + 'name': 'DAMN.', + 'artists': [{'id': 'kdot', 'name': 'Kendrick Lamar'}], + 'release_date': '2017-04-14', + 'total_tracks': 14, + 'album_type': 'album', + 'images': [{'url': 'https://i.scdn.co/640.jpg', 'height': 640}], + }, +} + + +def test_typed_path_used_for_known_source(): + payload = resolution._build_single_import_context_payload( + SAMPLE_SPOTIFY_TRACK, + source='spotify', + source_priority=['spotify'], + requested_title='HUMBLE.', + requested_artist='Kendrick Lamar', + ) + album = payload['context']['album'] + assert album['id'] == 'sp_album' + assert album['name'] == 'DAMN.' + assert album['release_date'] == '2017-04-14' + assert album['total_tracks'] == 14 + assert album['album_type'] == 'album' + assert album['image_url'] == 'https://i.scdn.co/640.jpg' + assert len(album['images']) == 1 + + +def test_typed_path_preserves_full_images_list(): + track = dict(SAMPLE_SPOTIFY_TRACK) + track['album'] = dict(SAMPLE_SPOTIFY_TRACK['album']) + track['album']['images'] = [ + {'url': 'https://i.scdn.co/640.jpg', 'height': 640}, + {'url': 'https://i.scdn.co/300.jpg', 'height': 300}, + ] + payload = resolution._build_single_import_context_payload( + track, source='spotify', source_priority=['spotify'], + ) + images = payload['context']['album']['images'] + assert len(images) == 2 + assert images[0]['url'] == 'https://i.scdn.co/640.jpg' + assert images[1]['url'] == 'https://i.scdn.co/300.jpg' + + +def test_legacy_path_used_when_no_source(): + payload = resolution._build_single_import_context_payload( + SAMPLE_SPOTIFY_TRACK, + source=None, + source_priority=[], + ) + album = payload['context']['album'] + assert album['id'] == 'sp_album' + assert album['name'] == 'DAMN.' + + +def test_legacy_path_used_for_unknown_source(): + payload = resolution._build_single_import_context_payload( + SAMPLE_SPOTIFY_TRACK, + source='made_up_provider', + source_priority=['made_up_provider'], + ) + album = payload['context']['album'] + assert album['id'] == 'sp_album' + assert album['name'] == 'DAMN.' + + +def test_legacy_path_used_when_typed_converter_raises(): + def _exploding(_): + raise RuntimeError('simulated converter bug') + + with patch.dict(resolution._TYPED_ALBUM_CONVERTERS, + {'spotify': _exploding}): + payload = resolution._build_single_import_context_payload( + SAMPLE_SPOTIFY_TRACK, + source='spotify', + source_priority=['spotify'], + ) + album = payload['context']['album'] + # Legacy path resolves the album fields successfully. + assert album['id'] == 'sp_album' + assert album['name'] == 'DAMN.' + + +@pytest.mark.parametrize('source,track', [ + ('itunes', { + 'id': 1, 'name': 'X', 'artists': [{'name': 'Y'}], + 'album': {'collectionId': 99, 'collectionName': 'iTunes Album', + 'artistName': 'Artist', 'trackCount': 10}, + }), + ('deezer', { + 'id': 2, 'name': 'X', 'artists': [{'name': 'Y'}], + 'album': {'id': 99, 'title': 'Deezer Album', + 'artist': {'name': 'Artist'}, 'nb_tracks': 8}, + }), + ('discogs', { + 'id': 3, 'name': 'X', 'artists': [{'name': 'Y'}], + 'album': {'id': 99, 'title': 'Discogs Album', + 'artists': [{'name': 'Artist'}], 'year': 2020}, + }), +]) +def test_typed_path_works_for_other_providers(source, track): + payload = resolution._build_single_import_context_payload( + track, source=source, source_priority=[source], + ) + album = payload['context']['album'] + assert album['name'] # populated by the typed converter + assert album['id'] diff --git a/tests/imports/test_file_integrity.py b/tests/imports/test_file_integrity.py new file mode 100644 index 00000000..0d56b531 --- /dev/null +++ b/tests/imports/test_file_integrity.py @@ -0,0 +1,254 @@ +"""Regression tests for file integrity checks on downloaded audio. + +Discord-reported (fresh.dumbledore [VRN]): slskd sometimes hosts broken +files (truncated transfers, corrupted FLAC, wrong file masquerading as +the target). The integrity layer at ``core/imports/file_integrity.py`` +catches these before they reach tagging/library sync, using three +universal checks: file-size sanity, mutagen parse, and duration +agreement against the metadata-source-provided expected length. + +These tests exercise the module directly with fabricated files (real +mp3 + flac samples generated via mutagen-friendly stubs and a couple of +hand-written WAV/FLAC files) so we don't need ffmpeg or live downloads. +""" + +from __future__ import annotations + +import os +import struct +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from core.imports import file_integrity + + +def _write_minimal_wav(path: Path, duration_s: float = 1.0, sample_rate: int = 8000) -> None: + """Write a minimal valid WAV file. Mutagen parses WAV via the + standard wave module wrapper, giving us a real `info.length` + we can assert against without needing ffmpeg.""" + n_samples = int(duration_s * sample_rate) + n_channels = 1 + bits_per_sample = 16 + byte_rate = sample_rate * n_channels * bits_per_sample // 8 + block_align = n_channels * bits_per_sample // 8 + data_size = n_samples * block_align + fmt_chunk = struct.pack( + "<4sIHHIIHH", + b"fmt ", 16, 1, n_channels, sample_rate, byte_rate, block_align, bits_per_sample, + ) + data_chunk = struct.pack("<4sI", b"data", data_size) + (b"\x00\x00" * n_samples) + riff = struct.pack("<4sI4s", b"RIFF", 4 + len(fmt_chunk) + len(data_chunk), b"WAVE") + path.write_bytes(riff + fmt_chunk + data_chunk) + + +# --------------------------------------------------------------------------- +# File size check +# --------------------------------------------------------------------------- + + +def test_rejects_zero_byte_file(tmp_path: Path) -> None: + """A 0-byte file is the most common slskd-broken case.""" + f = tmp_path / "empty.flac" + f.write_bytes(b"") + + result = file_integrity.check_audio_integrity(str(f)) + + assert result.ok is False + assert "too small" in result.reason.lower() + assert result.checks["size_bytes"] == 0 + + +def test_rejects_tiny_stub(tmp_path: Path) -> None: + """A few hundred bytes can't be a real audio file — slskd dropped a stub.""" + f = tmp_path / "stub.mp3" + f.write_bytes(b"x" * 500) + + result = file_integrity.check_audio_integrity(str(f)) + + assert result.ok is False + assert "too small" in result.reason.lower() + + +def test_size_threshold_is_overridable(tmp_path: Path) -> None: + """Tests / dev workflows can lower the size threshold.""" + f = tmp_path / "small_but_intentional.bin" + f.write_bytes(b"y" * 100) + + # Should pass the size check at threshold=50, then fail mutagen parse + # since it's not real audio. + result = file_integrity.check_audio_integrity(str(f), min_file_size_bytes=50) + + assert result.ok is False + assert "mutagen" in result.reason.lower() or "could not" in result.reason.lower() + + +def test_missing_file_returns_clean_failure(tmp_path: Path) -> None: + """No exception should escape — caller wants a clean boolean.""" + result = file_integrity.check_audio_integrity(str(tmp_path / "ghost.flac")) + + assert result.ok is False + assert "stat" in result.reason.lower() or "cannot" in result.reason.lower() + + +# --------------------------------------------------------------------------- +# Mutagen parse check +# --------------------------------------------------------------------------- + + +def test_rejects_non_audio_file_with_audio_extension(tmp_path: Path) -> None: + """A text file renamed to .flac (sometimes happens when slskd matches + a wrong file) should fail the parse check, not slip through.""" + f = tmp_path / "fake.flac" + # Big enough to clear the size check, but not audio. + f.write_bytes(b"this is definitely not flac data\n" * 1000) + + result = file_integrity.check_audio_integrity(str(f)) + + assert result.ok is False + # Either mutagen returns None (unidentified) or raises — either is a fail. + assert "mutagen" in result.reason.lower() or "no info" in result.reason.lower() or "identify" in result.reason.lower() + + +def test_accepts_valid_wav_with_no_expected_duration(tmp_path: Path) -> None: + """Real audio with no caller-provided duration should pass — only + size + parse run.""" + f = tmp_path / "real.wav" + _write_minimal_wav(f, duration_s=2.0) + + result = file_integrity.check_audio_integrity(str(f)) + + assert result.ok is True + assert result.checks["actual_length_s"] == pytest.approx(2.0, abs=0.1) + assert result.checks["length_check"] == "skipped" + + +# --------------------------------------------------------------------------- +# Duration agreement check +# --------------------------------------------------------------------------- + + +def test_accepts_when_length_within_tolerance(tmp_path: Path) -> None: + """A 5-second file claiming 5.5 seconds (within 3s tolerance) passes.""" + f = tmp_path / "track.wav" + _write_minimal_wav(f, duration_s=5.0) + + result = file_integrity.check_audio_integrity(str(f), expected_duration_ms=5500) + + assert result.ok is True + assert result.checks["length_check"] == "passed" + assert result.checks["length_drift_s"] == pytest.approx(0.5, abs=0.2) + + +def test_rejects_truncated_file(tmp_path: Path) -> None: + """A 2-second file claiming to be a 30-second track is truncated. + This is the headline slskd case — bytes stopped flowing partway + through but slskd reported success.""" + f = tmp_path / "truncated.wav" + _write_minimal_wav(f, duration_s=2.0) + + result = file_integrity.check_audio_integrity(str(f), expected_duration_ms=30_000) + + assert result.ok is False + assert "duration" in result.reason.lower() or "drift" in result.reason.lower() + assert result.checks["length_drift_s"] > 3.0 + + +def test_rejects_wrong_file_substituted(tmp_path: Path) -> None: + """A 10-second clip masquerading as a 3-minute album track. slskd + matched on a similar filename but the actual content is a snippet.""" + f = tmp_path / "wrong.wav" + _write_minimal_wav(f, duration_s=10.0) + + result = file_integrity.check_audio_integrity(str(f), expected_duration_ms=180_000) + + assert result.ok is False + assert result.checks["length_drift_s"] > 100 + + +def test_long_track_uses_wider_tolerance(tmp_path: Path) -> None: + """Tracks over 10 minutes get 5s tolerance instead of 3s — long + tracks naturally drift more (intros, outros, encoder padding).""" + # Write a 12-minute file (720s) but at minimum sample rate to keep + # the test fast — under 30KB total. + f = tmp_path / "long.wav" + _write_minimal_wav(f, duration_s=720.0, sample_rate=8000) + + # Claim 724 seconds — 4s drift, which would fail the 3s default but + # passes the 5s long-track threshold. + result = file_integrity.check_audio_integrity(str(f), expected_duration_ms=724_000) + + assert result.ok is True + assert result.checks["length_tolerance_s"] == pytest.approx(5.0) + + +def test_caller_can_override_tolerance(tmp_path: Path) -> None: + """Edge cases (e.g. live recordings, known-flaky sources) can opt + into a wider tolerance per-call.""" + f = tmp_path / "loose.wav" + _write_minimal_wav(f, duration_s=5.0) + + # 8-second drift — would fail default 3s, passes explicit 10s. + result = file_integrity.check_audio_integrity( + str(f), expected_duration_ms=13_000, length_tolerance_s=10.0, + ) + + assert result.ok is True + + +def test_zero_expected_duration_skips_length_check(tmp_path: Path) -> None: + """Some metadata sources don't carry duration — duration check + must be skipped, not treated as a 0-length match.""" + f = tmp_path / "no_duration.wav" + _write_minimal_wav(f, duration_s=5.0) + + result = file_integrity.check_audio_integrity(str(f), expected_duration_ms=0) + + assert result.ok is True + assert result.checks["length_check"] == "skipped" + + +def test_negative_expected_duration_skips_length_check(tmp_path: Path) -> None: + """Defensive: bad metadata returning negative duration shouldn't + crash or false-reject.""" + f = tmp_path / "neg_duration.wav" + _write_minimal_wav(f, duration_s=5.0) + + result = file_integrity.check_audio_integrity(str(f), expected_duration_ms=-100) + + assert result.ok is True + assert result.checks["length_check"] == "skipped" + + +# --------------------------------------------------------------------------- +# Failure-mode robustness +# --------------------------------------------------------------------------- + + +def test_check_never_raises(tmp_path: Path, monkeypatch) -> None: + """The integrity check is wrapped in try/except in pipeline.py but + callers shouldn't have to. Verify that even pathological inputs + return a clean IntegrityResult.""" + f = tmp_path / "test.wav" + _write_minimal_wav(f, duration_s=2.0) + + # Force a mutagen import-time failure by stubbing the import. + # Should NOT raise — should pass gracefully (mutagen unavailable). + real_import = __builtins__["__import__"] if isinstance(__builtins__, dict) else __builtins__.__import__ + + def _broken_import(name, *args, **kwargs): + if name == "mutagen": + raise ImportError("simulated missing mutagen") + return real_import(name, *args, **kwargs) + + monkeypatch.setitem(__builtins__ if isinstance(__builtins__, dict) else __builtins__.__dict__, + "__import__", _broken_import) + + try: + result = file_integrity.check_audio_integrity(str(f)) + except Exception as e: + pytest.fail(f"check_audio_integrity raised: {e}") + + assert result.ok is True + assert result.checks.get("mutagen_parse") == "unavailable" diff --git a/tests/imports/test_import_context.py b/tests/imports/test_import_context.py index 65ab1e12..e71584b7 100644 --- a/tests/imports/test_import_context.py +++ b/tests/imports/test_import_context.py @@ -206,6 +206,35 @@ def test_get_import_source_ids_prefers_nested_source_specific_ids(): } +def test_get_import_source_ids_prefers_spotify_ids_over_numeric_fallbacks(): + context = normalize_import_context( + { + "source": "spotify", + "artist": { + "id": "396753", + "spotify_artist_id": "sp-artist-1", + "deezer_id": "396753", + }, + "album": { + "id": "284076172", + "spotify_album_id": "sp-album-1", + "deezer_id": "284076172", + }, + "track_info": { + "id": "1607091752", + "spotify_track_id": "sp-track-1", + "deezer_id": "1607091752", + }, + } + ) + + assert get_import_source_ids(context) == { + "track_id": "sp-track-1", + "artist_id": "sp-artist-1", + "album_id": "sp-album-1", + } + + def test_build_import_album_info_uses_normalized_album_context(): context = normalize_import_context( { diff --git a/tests/imports/test_import_pipeline.py b/tests/imports/test_import_pipeline.py index 51b34361..98159d0f 100644 --- a/tests/imports/test_import_pipeline.py +++ b/tests/imports/test_import_pipeline.py @@ -85,6 +85,12 @@ def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): fake_acoustid.VerificationResult = types.SimpleNamespace(FAIL="FAIL") monkeypatch.setitem(sys.modules, "core.acoustid_verification", fake_acoustid) + # The integrity layer would reject these 5-byte fixture files; bypass + # it since these tests cover plumbing (notification + metadata_runtime + # forwarding), not integrity behavior. + from core.imports.file_integrity import IntegrityResult + monkeypatch.setattr(import_pipeline, "check_audio_integrity", + lambda *_a, **_kw: IntegrityResult(ok=True, checks={"size_bytes": 5, "actual_length_s": 0})) monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config(str(transfer_root))) monkeypatch.setattr(import_pipeline, "add_activity_item", lambda *args, **kwargs: activity_calls.append((args, kwargs))) monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None) @@ -175,6 +181,11 @@ def test_post_process_matched_download_forwards_separate_metadata_runtime(tmp_pa monkeypatch.setattr(import_pipeline, "get_import_clean_title", lambda *args, **kwargs: "Track") monkeypatch.setattr(import_pipeline, "get_audio_quality_string", lambda file_path: "") monkeypatch.setattr(import_pipeline, "check_flac_bit_depth", lambda *args, **kwargs: None) + # Bypass integrity check — the 5-byte fixture would fail it; this test + # exercises the metadata-runtime forwarding path, not file integrity. + from core.imports.file_integrity import IntegrityResult + monkeypatch.setattr(import_pipeline, "check_audio_integrity", + lambda *_a, **_kw: IntegrityResult(ok=True, checks={})) monkeypatch.setattr(import_pipeline, "build_final_path_for_track", lambda *args, **kwargs: (str(target_path), None)) def _capture_enhance(file_path, context, artist, album_info, runtime=None): diff --git a/tests/imports/test_import_side_effects.py b/tests/imports/test_import_side_effects.py index 697d845b..33bfa1e4 100644 --- a/tests/imports/test_import_side_effects.py +++ b/tests/imports/test_import_side_effects.py @@ -1,3 +1,4 @@ +import os import sqlite3 from types import SimpleNamespace @@ -58,6 +59,7 @@ def _make_soulsync_db(): duration INTEGER, file_path TEXT, bitrate INTEGER, + file_size INTEGER, track_artist TEXT, server_source TEXT, created_at TEXT, @@ -142,3 +144,63 @@ def test_record_soulsync_library_entry_writes_artist_album_and_track(tmp_path, m assert track_row["track_artist"] == "Guest Artist" assert track_row["album_id"] == album_row["id"] assert track_row["file_path"] == str(final_path) + # File size in bytes — populates the Library Disk Usage card on Stats. + # Read via os.path.getsize at insert time since SoulSync standalone is + # the only flow where the file is local at the moment we write the row. + assert track_row["file_size"] == os.path.getsize(str(final_path)) + + +def test_record_soulsync_library_entry_ignores_numeric_spotify_ids(tmp_path, monkeypatch): + conn = _make_soulsync_db() + fake_db = _FakeDB(conn) + final_path = tmp_path / "track.flac" + final_path.write_bytes(b"audio") + + monkeypatch.setattr(side_effects, "get_database", lambda: fake_db) + monkeypatch.setattr( + side_effects, + "_get_config_manager", + lambda: SimpleNamespace(get_active_media_server=lambda: "soulsync"), + ) + + import core.genre_filter as genre_filter + + monkeypatch.setattr(genre_filter, "filter_genres", lambda genres, _cfg: genres) + + context = { + "source": "spotify", + "artist": {"id": "396753", "name": "Artist One"}, + "album": { + "id": "284076172", + "name": "Album One", + "release_date": "2024-02-03", + "total_tracks": 12, + }, + "track_info": { + "id": "1607091752", + "name": "Song One", + "track_number": 7, + "duration_ms": 210000, + "artists": [{"name": "Guest Artist"}], + "_source": "spotify", + }, + "original_search_result": { + "title": "Song One", + "artists": [{"name": "Guest Artist"}], + "_source": "spotify", + }, + "_final_processed_path": str(final_path), + } + + artist_context = {"name": "Artist One", "genres": ["rock"]} + album_info = {"is_album": True, "album_name": "Album One", "track_number": 7} + + side_effects.record_soulsync_library_entry(context, artist_context, album_info) + + artist_row = conn.execute("SELECT * FROM artists").fetchone() + album_row = conn.execute("SELECT * FROM albums").fetchone() + track_row = conn.execute("SELECT * FROM tracks").fetchone() + + assert artist_row["spotify_artist_id"] is None + assert album_row["spotify_album_id"] is None + assert track_row["spotify_track_id"] is None diff --git a/tests/imports/test_lossy_copy_delete_original.py b/tests/imports/test_lossy_copy_delete_original.py new file mode 100644 index 00000000..6089736e --- /dev/null +++ b/tests/imports/test_lossy_copy_delete_original.py @@ -0,0 +1,214 @@ +"""Regression tests for lossy_copy.delete_original honoring. + +Discord-reported (CAL): with ``lossy_copy.enabled=True``, +``lossy_copy.delete_original=True``, and ``codec=mp3``, downloads +ended up with BOTH the original FLAC AND the converted MP3 in the +target folder. The setting was being read by the pre-move source- +vanished check at ``core/imports/pipeline.py`` but never acted on +during the actual conversion step. Result: a "lossy-only" library +ended up dual-format on every import. + +These tests pin the behavior so the regression doesn't return — they +exercise ``create_lossy_copy`` directly with ffmpeg stubbed via +monkeypatch, asserting the original is deleted only when the setting +is enabled and the conversion succeeded. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from core.imports import file_ops + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_flac(tmp_path: Path) -> Path: + """A placeholder FLAC file — content doesn't matter, ffmpeg call is stubbed.""" + src = tmp_path / "01 - Track.flac" + src.write_bytes(b"FAKE-FLAC-CONTENT") + return src + + +def _stub_config(monkeypatch, **overrides): + """Patch the file_ops module's config_manager so each test controls + only the keys it cares about.""" + defaults = { + "lossy_copy.enabled": True, + "lossy_copy.codec": "mp3", + "lossy_copy.bitrate": "320", + "lossy_copy.delete_original": False, + } + defaults.update(overrides) + + fake_cfg = MagicMock() + fake_cfg.get.side_effect = lambda key, default=None: defaults.get(key, default) + monkeypatch.setattr(file_ops, "config_manager", fake_cfg) + return defaults + + +def _stub_ffmpeg_success(monkeypatch, fake_flac: Path): + """Stub shutil.which to report ffmpeg available + subprocess.run to + write a fake MP3 to out_path and return success.""" + monkeypatch.setattr(file_ops.shutil, "which", lambda _: "/fake/ffmpeg") + + def _fake_run(cmd, **_kwargs): + # cmd[-1] is the out_path (per ffmpeg invocation in create_lossy_copy) + out_path = cmd[-1] + Path(out_path).write_bytes(b"FAKE-MP3-CONTENT") + return SimpleNamespace(returncode=0, stderr="", stdout="") + + monkeypatch.setattr(file_ops.subprocess, "run", _fake_run) + + # Skip the mutagen tagging step — file is fake bytes, mutagen would + # raise. Accepting the silent-fail path is fine here; tests assert + # on file presence, not tag content. + monkeypatch.setattr( + "mutagen.File", + lambda _path: None, + raising=False, + ) + + +def _stub_ffmpeg_failure(monkeypatch): + """Stub ffmpeg to return non-zero so the conversion path bails out.""" + monkeypatch.setattr(file_ops.shutil, "which", lambda _: "/fake/ffmpeg") + monkeypatch.setattr( + file_ops.subprocess, + "run", + lambda cmd, **kw: SimpleNamespace(returncode=1, stderr="fake ffmpeg error", stdout=""), + ) + + +# --------------------------------------------------------------------------- +# delete_original honored after successful conversion +# --------------------------------------------------------------------------- + + +class TestDeleteOriginalHonored: + def test_original_flac_removed_when_setting_enabled(self, monkeypatch, fake_flac: Path): + _stub_config(monkeypatch, **{"lossy_copy.delete_original": True}) + _stub_ffmpeg_success(monkeypatch, fake_flac) + + out_path = file_ops.create_lossy_copy(str(fake_flac)) + + assert out_path is not None + assert out_path.endswith(".mp3") + assert Path(out_path).exists(), "MP3 should have been written" + assert not fake_flac.exists(), \ + "Original FLAC must be removed when lossy_copy.delete_original=True" + + def test_original_flac_kept_when_setting_disabled(self, monkeypatch, fake_flac: Path): + _stub_config(monkeypatch, **{"lossy_copy.delete_original": False}) + _stub_ffmpeg_success(monkeypatch, fake_flac) + + out_path = file_ops.create_lossy_copy(str(fake_flac)) + + assert out_path is not None + assert Path(out_path).exists() + assert fake_flac.exists(), \ + "Original FLAC must survive when lossy_copy.delete_original=False" + + def test_default_is_keep_original(self, monkeypatch, fake_flac: Path): + """When the user never set the option, default = keep original. + Defensive: a missing config value must not silently drop files.""" + # Don't override delete_original — picks up the default in _stub_config (False) + _stub_config(monkeypatch) + _stub_ffmpeg_success(monkeypatch, fake_flac) + + file_ops.create_lossy_copy(str(fake_flac)) + assert fake_flac.exists() + + +# --------------------------------------------------------------------------- +# delete_original NOT triggered when conversion fails +# --------------------------------------------------------------------------- + + +class TestDeleteOriginalSkippedOnFailure: + def test_original_kept_when_ffmpeg_fails(self, monkeypatch, fake_flac: Path): + """If ffmpeg returns non-zero, the conversion is treated as failed + and the original must NOT be deleted (would leave the user with + no audio file at all).""" + _stub_config(monkeypatch, **{"lossy_copy.delete_original": True}) + _stub_ffmpeg_failure(monkeypatch) + + out_path = file_ops.create_lossy_copy(str(fake_flac)) + + assert out_path is None, "Conversion failure must return None" + assert fake_flac.exists(), \ + "Original FLAC must survive a failed conversion regardless of delete_original" + + def test_original_kept_when_lossy_copy_disabled(self, monkeypatch, fake_flac: Path): + """The function early-returns when lossy_copy.enabled=False — so + delete_original cannot fire even if it's enabled.""" + _stub_config(monkeypatch, **{ + "lossy_copy.enabled": False, + "lossy_copy.delete_original": True, + }) + _stub_ffmpeg_success(monkeypatch, fake_flac) + + result = file_ops.create_lossy_copy(str(fake_flac)) + assert result is None + assert fake_flac.exists() + + +# --------------------------------------------------------------------------- +# Defensive paths +# --------------------------------------------------------------------------- + + +class TestDeleteOriginalDefensive: + def test_does_not_crash_when_original_already_gone(self, monkeypatch, fake_flac: Path): + """If something else (concurrent worker, dedup cleanup) removed + the original between conversion and deletion, that's not an + error — we just got our wish slightly early.""" + _stub_config(monkeypatch, **{"lossy_copy.delete_original": True}) + _stub_ffmpeg_success(monkeypatch, fake_flac) + + original_remove = os.remove + + def _remove_after_unlinking_first(path, *args, **kwargs): + # Simulate the source being gone before the delete call: pre- + # remove on first call, then defer to real os.remove. + if Path(path) == fake_flac and fake_flac.exists(): + fake_flac.unlink() + # Now raise FileNotFoundError as os.remove would on a missing path + raise FileNotFoundError(2, "No such file or directory", str(path)) + return original_remove(path, *args, **kwargs) + + monkeypatch.setattr(file_ops.os, "remove", _remove_after_unlinking_first) + + out_path = file_ops.create_lossy_copy(str(fake_flac)) + assert out_path is not None + assert Path(out_path).exists() + assert not fake_flac.exists() + + def test_handles_oserror_during_delete_without_propagating(self, monkeypatch, fake_flac: Path): + """If the actual unlink fails (permission error, locked file, FS + full), the conversion is still considered successful — the lossy + copy already exists. We log the error but return the out_path so + the import pipeline can continue. The original is left in place + for the user to clean up manually.""" + _stub_config(monkeypatch, **{"lossy_copy.delete_original": True}) + _stub_ffmpeg_success(monkeypatch, fake_flac) + + def _failing_remove(path, *args, **kwargs): + raise PermissionError(13, "Permission denied", str(path)) + + monkeypatch.setattr(file_ops.os, "remove", _failing_remove) + + out_path = file_ops.create_lossy_copy(str(fake_flac)) + assert out_path is not None, "Failed delete must not break conversion return value" + assert Path(out_path).exists() + assert fake_flac.exists(), "Failed delete leaves the original in place" diff --git a/tests/library/test_path_resolver.py b/tests/library/test_path_resolver.py new file mode 100644 index 00000000..9c32e9e3 --- /dev/null +++ b/tests/library/test_path_resolver.py @@ -0,0 +1,405 @@ +"""Regression tests for ``core/library/path_resolver.py``. + +GitHub issue #476 (gabistek, Docker / Arch host): Album Completeness +Auto-Fill returned ``Could not determine album folder from existing +tracks`` for every album. Root cause: the repair worker's path +resolver only probed the transfer + download folders, not the +user-configured ``library.music_paths`` or Plex-reported library +locations. Files lived in the media-server library mount and got +silently treated as missing. + +These tests pin the resolver's behavior across the four base-dir +sources (explicit transfer, explicit download, config-driven library +paths, Plex client locations), the suffix-walk algorithm, and the +defensive return-None paths. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from core.library import path_resolver +from core.library.path_resolver import resolve_library_file_path + + +# --------------------------------------------------------------------------- +# Defensive boundary cases +# --------------------------------------------------------------------------- + + +def test_returns_none_for_empty_path() -> None: + assert resolve_library_file_path("") is None + assert resolve_library_file_path(None) is None # type: ignore[arg-type] + + +def test_returns_none_when_no_base_dirs_configured(tmp_path: Path) -> None: + """No transfer, no download, no config, no Plex → can't resolve.""" + fake = tmp_path / "non_existent.flac" + assert resolve_library_file_path(str(fake)) is None + + +def test_returns_raw_path_when_it_exists(tmp_path: Path) -> None: + """Happy path — the raw stored path resolves directly.""" + real = tmp_path / "track.flac" + real.write_bytes(b"audio") + + result = resolve_library_file_path(str(real)) + + assert result == str(real) + + +# --------------------------------------------------------------------------- +# Transfer / download base dirs (legacy behavior preserved) +# --------------------------------------------------------------------------- + + +def test_finds_file_via_transfer_folder_suffix_walk(tmp_path: Path) -> None: + """DB stored path is `/music/Artist/Album/track.flac` but the file + actually lives in `/Artist/Album/track.flac`.""" + transfer = tmp_path / "Transfer" + (transfer / "Artist" / "Album").mkdir(parents=True) + actual = transfer / "Artist" / "Album" / "track.flac" + actual.write_bytes(b"audio") + + result = resolve_library_file_path( + "/music/Artist/Album/track.flac", + transfer_folder=str(transfer), + ) + + assert result == str(actual) + + +def test_finds_file_via_download_folder_when_transfer_misses(tmp_path: Path) -> None: + transfer = tmp_path / "Transfer" + transfer.mkdir() + download = tmp_path / "Downloads" + (download / "Artist" / "Album").mkdir(parents=True) + actual = download / "Artist" / "Album" / "track.flac" + actual.write_bytes(b"audio") + + result = resolve_library_file_path( + "/music/Artist/Album/track.flac", + transfer_folder=str(transfer), + download_folder=str(download), + ) + + assert result == str(actual) + + +def test_handles_windows_backslash_paths(tmp_path: Path) -> None: + """DB stored paths can be Windows-style with backslashes — the + walker normalizes them to forward slashes before splitting.""" + transfer = tmp_path / "Transfer" + (transfer / "Artist" / "Album").mkdir(parents=True) + actual = transfer / "Artist" / "Album" / "track.flac" + actual.write_bytes(b"audio") + + result = resolve_library_file_path( + r"H:\Music\Artist\Album\track.flac", + transfer_folder=str(transfer), + ) + + assert result == str(actual) + + +# --------------------------------------------------------------------------- +# Library music paths from config (the fix for #476) +# --------------------------------------------------------------------------- + + +def test_finds_file_via_library_music_paths(tmp_path: Path) -> None: + """The Plex/Jellyfin library at /Artist/Album/track.flac + is found via the user's configured ``library.music_paths`` even + when transfer + download don't have it.""" + transfer = tmp_path / "Transfer" + transfer.mkdir() + library = tmp_path / "Library" + (library / "Artist" / "Album").mkdir(parents=True) + actual = library / "Artist" / "Album" / "track.flac" + actual.write_bytes(b"audio") + + cm = MagicMock() + cm.get.side_effect = lambda key, default=None: { + "library.music_paths": [str(library)], + "soulseek.transfer_path": "", + "soulseek.download_path": "", + }.get(key, default) + + result = resolve_library_file_path( + "/data/music/Artist/Album/track.flac", + transfer_folder=str(transfer), + config_manager=cm, + ) + + assert result == str(actual) + + +def test_library_music_paths_handles_multiple_dirs(tmp_path: Path) -> None: + """Users can configure multiple music paths — first existing + suffix-match wins.""" + lib_a = tmp_path / "LibA" + lib_b = tmp_path / "LibB" + (lib_a / "Artist").mkdir(parents=True) + (lib_b / "Artist" / "Album").mkdir(parents=True) + # Only LibB has the actual file + actual = lib_b / "Artist" / "Album" / "track.flac" + actual.write_bytes(b"audio") + + cm = MagicMock() + cm.get.side_effect = lambda key, default=None: { + "library.music_paths": [str(lib_a), str(lib_b)], + }.get(key, default) + + result = resolve_library_file_path( + "/x/Artist/Album/track.flac", + config_manager=cm, + ) + + assert result == str(actual) + + +def test_skips_non_string_entries_in_music_paths(tmp_path: Path) -> None: + """Defensive: malformed config (None, int, dict in the list) must + not crash the resolver.""" + library = tmp_path / "Library" + (library / "Artist").mkdir(parents=True) + actual = library / "Artist" / "track.flac" + actual.write_bytes(b"audio") + + cm = MagicMock() + cm.get.side_effect = lambda key, default=None: { + "library.music_paths": [None, 42, {"x": 1}, str(library), ""], + }.get(key, default) + + result = resolve_library_file_path( + "/x/Artist/track.flac", + config_manager=cm, + ) + + assert result == str(actual) + + +def test_strips_whitespace_from_music_paths(tmp_path: Path) -> None: + """Trailing whitespace on a config value (common copy-paste mistake) + shouldn't break resolution.""" + library = tmp_path / "Library" + (library / "Artist").mkdir(parents=True) + actual = library / "Artist" / "track.flac" + actual.write_bytes(b"audio") + + cm = MagicMock() + cm.get.side_effect = lambda key, default=None: { + "library.music_paths": [f" {library} "], + }.get(key, default) + + result = resolve_library_file_path( + "/x/Artist/track.flac", + config_manager=cm, + ) + + assert result == str(actual) + + +# --------------------------------------------------------------------------- +# Plex-reported library locations +# --------------------------------------------------------------------------- + + +def test_finds_file_via_plex_library_location(tmp_path: Path) -> None: + """When SoulSync mounts the Plex library at a different path than + Plex itself reports, the Plex-reported location is added to the + search and the file is found.""" + plex_loc = tmp_path / "PlexLibrary" + (plex_loc / "Artist" / "Album").mkdir(parents=True) + actual = plex_loc / "Artist" / "Album" / "track.flac" + actual.write_bytes(b"audio") + + plex_client = SimpleNamespace( + server=SimpleNamespace(), # truthy + music_library=SimpleNamespace(locations=[str(plex_loc)]), + ) + + result = resolve_library_file_path( + "/music/Artist/Album/track.flac", + plex_client=plex_client, + ) + + assert result == str(actual) + + +def test_handles_plex_client_without_server(tmp_path: Path) -> None: + """Plex client with no `server` attribute (uninitialized) shouldn't + crash — just skip the Plex source.""" + plex_client = SimpleNamespace(server=None, music_library=None) + + # No other sources configured → returns None, no exception. + assert resolve_library_file_path( + "/x/track.flac", + plex_client=plex_client, + ) is None + + +def test_handles_plex_locations_attribute_missing(tmp_path: Path) -> None: + """Plex music_library object without a `locations` attribute → skip.""" + plex_client = SimpleNamespace( + server=SimpleNamespace(), + music_library=SimpleNamespace(), # no `locations` + ) + + assert resolve_library_file_path( + "/x/track.flac", + plex_client=plex_client, + ) is None + + +# --------------------------------------------------------------------------- +# Source ordering & deduplication +# --------------------------------------------------------------------------- + + +def test_dedupe_avoids_duplicate_probes(tmp_path: Path) -> None: + """If transfer == library_paths[0], the dir is only probed once. + Resolver still finds the file.""" + shared = tmp_path / "Shared" + (shared / "Artist").mkdir(parents=True) + actual = shared / "Artist" / "track.flac" + actual.write_bytes(b"audio") + + cm = MagicMock() + cm.get.side_effect = lambda key, default=None: { + "soulseek.transfer_path": str(shared), + "library.music_paths": [str(shared)], + }.get(key, default) + + result = resolve_library_file_path( + "/x/Artist/track.flac", + transfer_folder=str(shared), + config_manager=cm, + ) + + assert result == str(actual) + + +def test_explicit_transfer_takes_priority_over_config(tmp_path: Path) -> None: + """Explicit transfer kwarg is added before config-derived paths so + the worker's already-cached transfer_folder always wins ties.""" + explicit = tmp_path / "Explicit" + config_dir = tmp_path / "FromConfig" + (explicit / "Artist").mkdir(parents=True) + (config_dir / "Artist").mkdir(parents=True) + actual_explicit = explicit / "Artist" / "track.flac" + actual_config = config_dir / "Artist" / "track.flac" + actual_explicit.write_bytes(b"explicit") + actual_config.write_bytes(b"config") + + cm = MagicMock() + cm.get.side_effect = lambda key, default=None: { + "soulseek.transfer_path": str(config_dir), + }.get(key, default) + + result = resolve_library_file_path( + "/x/Artist/track.flac", + transfer_folder=str(explicit), + config_manager=cm, + ) + + assert result == str(actual_explicit) + + +# --------------------------------------------------------------------------- +# Failure paths from external dependencies don't crash +# --------------------------------------------------------------------------- + + +def test_config_manager_get_raising_does_not_crash(tmp_path: Path) -> None: + """A flaky config_manager.get raising shouldn't break resolution. + The resolver swallows it and continues with the explicit dirs.""" + transfer = tmp_path / "Transfer" + (transfer / "Artist").mkdir(parents=True) + actual = transfer / "Artist" / "track.flac" + actual.write_bytes(b"audio") + + cm = MagicMock() + cm.get.side_effect = RuntimeError("config blew up") + + result = resolve_library_file_path( + "/x/Artist/track.flac", + transfer_folder=str(transfer), + config_manager=cm, + ) + + assert result == str(actual) + + +def test_plex_client_attribute_access_raising_does_not_crash(tmp_path: Path) -> None: + """A Plex client whose attribute access raises shouldn't break + resolution — fallback to other sources.""" + transfer = tmp_path / "Transfer" + (transfer / "Artist").mkdir(parents=True) + actual = transfer / "Artist" / "track.flac" + actual.write_bytes(b"audio") + + class _BrokenPlex: + @property + def server(self): + raise RuntimeError("plex disconnected") + + result = resolve_library_file_path( + "/x/Artist/track.flac", + transfer_folder=str(transfer), + plex_client=_BrokenPlex(), + ) + + assert result == str(actual) + + +def test_returns_none_when_no_suffix_matches(tmp_path: Path) -> None: + """When the file genuinely doesn't exist anywhere, return None + cleanly. Don't false-match an unrelated file.""" + transfer = tmp_path / "Transfer" + (transfer / "Artist" / "Album").mkdir(parents=True) + # Create a different file in the right tree + (transfer / "Artist" / "Album" / "different.flac").write_bytes(b"x") + + result = resolve_library_file_path( + "/x/Artist/Album/missing.flac", + transfer_folder=str(transfer), + ) + + assert result is None + + +# --------------------------------------------------------------------------- +# docker_resolve_path internal helper +# --------------------------------------------------------------------------- + + +def test_docker_resolve_path_translates_windows_paths_inside_docker(monkeypatch) -> None: + """Inside Docker, ``H:\\Music\\track.flac`` becomes + ``/host/mnt/h/Music/track.flac`` so the bind-mounted host drive + can be reached. Outside Docker, paths are returned unchanged.""" + real_exists = os.path.exists + + def _fake_exists(p): + if p == "/.dockerenv": + return True + return real_exists(p) + + monkeypatch.setattr(os.path, "exists", _fake_exists) + assert path_resolver._docker_resolve_path("H:\\Music\\track.flac") == "/host/mnt/h/Music/track.flac" + + # Non-Windows-style path passes through unchanged inside Docker too. + assert path_resolver._docker_resolve_path("/data/music") == "/data/music" + + +def test_docker_resolve_path_pass_through_outside_docker(monkeypatch) -> None: + """Outside Docker (no /.dockerenv), Windows paths are unchanged.""" + real_exists = os.path.exists + monkeypatch.setattr(os.path, "exists", + lambda p: False if p == "/.dockerenv" else real_exists(p)) + assert path_resolver._docker_resolve_path("H:\\Music\\track.flac") == "H:\\Music\\track.flac" diff --git a/tests/media_server/__init__.py b/tests/media_server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/media_server/test_conformance.py b/tests/media_server/test_conformance.py new file mode 100644 index 00000000..dcf789c6 --- /dev/null +++ b/tests/media_server/test_conformance.py @@ -0,0 +1,72 @@ +"""Pin the structural conformance of every media server client to +``MediaServerClient``. Mirrors the download plugin conformance test +shape — class-level checks (not instance-level) so importing the +test doesn't drag every server's heavy auth init into the test +collection phase. +""" + +from __future__ import annotations + +import pytest + +from core.media_server.contract import REQUIRED_METHODS + + +def _import_server_classes(): + """Import every server client class lazily inside tests so + auth-init-heavy modules (Plex, Jellyfin) aren't imported at test + collection.""" + from core.jellyfin_client import JellyfinClient + from core.navidrome_client import NavidromeClient + from core.plex_client import PlexClient + from core.soulsync_client import SoulSyncClient + + return { + 'plex': PlexClient, + 'jellyfin': JellyfinClient, + 'navidrome': NavidromeClient, + 'soulsync': SoulSyncClient, + } + + +def test_default_registry_registers_all_four_servers(): + """Smoke check that the foundation registry knows about every + server SoulSync historically dispatched to.""" + from core.media_server.registry import build_default_registry + + registry = build_default_registry() + expected = {'plex', 'jellyfin', 'navidrome', 'soulsync'} + assert set(registry.names()) == expected + + +@pytest.mark.parametrize('server_name', ['plex', 'jellyfin', 'navidrome', 'soulsync']) +def test_server_class_has_all_required_methods(server_name): + """Every registered server class exposes every required protocol + method by name. Diagnostic-friendly: tells you WHICH method is + missing when a new server is added without all the required + methods.""" + classes = _import_server_classes() + cls = classes[server_name] + + missing = [m for m in REQUIRED_METHODS if not hasattr(cls, m)] + assert not missing, ( + f"{server_name} ({cls.__name__}) missing required methods: {missing}" + ) + + +@pytest.mark.parametrize('server_name', ['plex', 'jellyfin', 'navidrome', 'soulsync']) +def test_server_class_explicitly_inherits_contract(server_name): + """Per Cin's standard from the download refactor: clients must + explicitly inherit ``MediaServerClient`` so the contract conformance + is obvious from reading the class declaration. Structural + typing alone (which would still pass `hasattr` checks) leaves + the contract invisible to anyone reading the code — drift in a + future client class wouldn't fail at the contract boundary.""" + from core.media_server.contract import MediaServerClient + + classes = _import_server_classes() + cls = classes[server_name] + assert issubclass(cls, MediaServerClient), ( + f"{cls.__name__} must explicitly inherit MediaServerClient — " + f"structural typing isn't enough" + ) diff --git a/tests/media_server/test_engine.py b/tests/media_server/test_engine.py new file mode 100644 index 00000000..262fbe1e --- /dev/null +++ b/tests/media_server/test_engine.py @@ -0,0 +1,302 @@ +"""Tests for MediaServerEngine cross-server dispatch.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from core.media_server.engine import MediaServerEngine +from core.media_server.registry import MediaServerRegistry, ServerSpec + + +class _FakeClient: + """Stand-in client supporting all required + optional methods. + Tests selectively override per-test to assert dispatch behavior.""" + + def __init__(self, name='fake', connected=True): + self.name = name + self._connected = connected + self._artists = [] + self._album_ids = set() + self._search_results = [] + self._scan_triggered = False + + def is_connected(self): + return self._connected + + def ensure_connection(self): + return self._connected + + def get_all_artists(self): + return self._artists + + def get_all_album_ids(self): + return self._album_ids + + def search_tracks(self, title, artist, limit=15): + return self._search_results + + def trigger_library_scan(self): + self._scan_triggered = True + return True + + def is_library_scanning(self): + return False + + def get_library_stats(self): + return {'artists': 0, 'albums': 0, 'tracks': 0} + + def get_recently_added_albums(self, max_results=400): + return [] + + +@pytest.fixture +def make_engine(): + """Build an engine with mock clients. Test passes a dict of + name → client; engine wires them via a registry + custom + active-server resolver.""" + + def _make(clients_by_name, active='plex'): + registry = MediaServerRegistry() + for name, client in clients_by_name.items(): + registry.register(ServerSpec( + name=name, + factory=lambda c=client: c, + display_name=name.title(), + )) + return MediaServerEngine( + registry=registry, + active_server_resolver=lambda: active, + ) + + return _make + + +# --------------------------------------------------------------------------- +# Active-server resolution +# --------------------------------------------------------------------------- + + +def test_active_server_property_reflects_resolver(make_engine): + plex = _FakeClient('plex') + jelly = _FakeClient('jellyfin') + engine = make_engine({'plex': plex, 'jellyfin': jelly}, active='jellyfin') + assert engine.active_server == 'jellyfin' + assert engine.active_client() is jelly + + +def test_client_lookup_by_name(make_engine): + plex = _FakeClient('plex') + engine = make_engine({'plex': plex}, active='plex') + assert engine.client('plex') is plex + assert engine.client('made_up') is None + + +# --------------------------------------------------------------------------- +# Required-method dispatch +# --------------------------------------------------------------------------- + + +def test_is_connected_routes_to_active_client(make_engine): + plex = _FakeClient('plex', connected=True) + jelly = _FakeClient('jellyfin', connected=False) + engine = make_engine({'plex': plex, 'jellyfin': jelly}, active='jellyfin') + assert engine.is_connected() is False # follows jellyfin + engine = make_engine({'plex': plex, 'jellyfin': jelly}, active='plex') + assert engine.is_connected() is True # follows plex + + +def test_engine_is_connected_returns_false_when_active_client_failed_to_init(): + """When the active client failed to initialize (registry stored + None), the engine returns False instead of raising.""" + registry = MediaServerRegistry() + registry.register(ServerSpec( + name='broken', + factory=lambda: (_ for _ in ()).throw(RuntimeError("init failed")), + display_name='Broken', + )) + registry.initialize() # captures the exception + engine = MediaServerEngine(registry=registry, active_server_resolver=lambda: 'broken') + + assert engine.is_connected() is False + assert engine.active_client() is None + + +def test_is_connected_swallows_exception_from_client(make_engine): + """If the client's is_connected raises, engine returns False + instead of propagating — dashboard status indicators stay + responsive even if a server is misbehaving.""" + plex = _FakeClient('plex') + plex.is_connected = MagicMock(side_effect=RuntimeError("boom")) + engine = make_engine({'plex': plex}, active='plex') + + assert engine.is_connected() is False + + +# --------------------------------------------------------------------------- +# Generic accessors (Cin's standard from the download refactor) +# --------------------------------------------------------------------------- + + +def test_configured_clients_only_returns_connected_servers(make_engine): + """Replaces the legacy per-server `if X and X.is_connected(): ...` + chains in web_server.py. Single call returns the dict.""" + plex = _FakeClient('plex', connected=True) + jelly = _FakeClient('jellyfin', connected=False) + soulsync = _FakeClient('soulsync', connected=True) + engine = make_engine( + {'plex': plex, 'jellyfin': jelly, 'soulsync': soulsync}, + active='plex', + ) + result = engine.configured_clients() + assert set(result.keys()) == {'plex', 'soulsync'} + assert result['plex'] is plex + assert result['soulsync'] is soulsync + + +def test_configured_clients_skips_clients_whose_is_connected_raises(make_engine): + """Defensive: a single broken is_connected() must not crash the + iteration. Healthy clients still come back.""" + healthy = _FakeClient('plex', connected=True) + broken = _FakeClient('jellyfin') + broken.is_connected = MagicMock(side_effect=RuntimeError("boom")) + engine = make_engine({'plex': healthy, 'jellyfin': broken}, active='plex') + result = engine.configured_clients() + assert 'plex' in result + assert 'jellyfin' not in result + + +def test_reload_config_dispatches_to_named_server(make_engine): + """Generic dispatch — caller passes server name instead of + reaching for plex_client.reload_config() directly.""" + + class _ReloadablePlex(_FakeClient): + def __init__(self): + super().__init__('plex') + self.reload_called = False + + def reload_config(self): + self.reload_called = True + + plex = _ReloadablePlex() + soulsync = _FakeClient('soulsync') # No reload_config method + engine = make_engine({'plex': plex, 'soulsync': soulsync}, active='plex') + + assert engine.reload_config('plex') is True + assert plex.reload_called is True + + +def test_reload_config_skips_clients_without_method(make_engine): + """Servers that don't expose reload_config are skipped silently + (return True).""" + soulsync = _FakeClient('soulsync') + engine = make_engine({'soulsync': soulsync}, active='soulsync') + assert engine.reload_config('soulsync') is True + + +def test_reload_config_with_no_args_reloads_every_server(make_engine): + """When called with no name, hits every registered server that + exposes reload_config.""" + + class _ReloadableClient(_FakeClient): + def __init__(self, name): + super().__init__(name) + self.reload_called = False + + def reload_config(self): + self.reload_called = True + + plex = _ReloadableClient('plex') + jelly = _ReloadableClient('jellyfin') + engine = make_engine({'plex': plex, 'jellyfin': jelly}, active='plex') + + engine.reload_config() + assert plex.reload_called is True + assert jelly.reload_called is True + + +# --------------------------------------------------------------------------- +# Singleton factory (matches get_metadata_engine() / get_download_orchestrator()) +# --------------------------------------------------------------------------- + + +def test_get_media_server_engine_returns_set_singleton(make_engine): + """When set_media_server_engine has been called (web_server.py + does this at boot), get_media_server_engine returns the installed + instance instead of building a fresh one with the default registry.""" + from core.media_server.engine import ( + get_media_server_engine, + set_media_server_engine, + ) + + engine = make_engine({'plex': _FakeClient('plex')}, active='plex') + set_media_server_engine(engine) + try: + assert get_media_server_engine() is engine + finally: + set_media_server_engine(None) + + +# --------------------------------------------------------------------------- +# Empty-engine fallback (web_server.py boot resilience) +# --------------------------------------------------------------------------- + + +def test_engine_with_empty_clients_dict_is_safe_to_use(): + """web_server.py falls back to ``MediaServerEngine(clients={})`` if + full engine init raises — preserves the resilience the per-server + globals had pre-refactor (each one had its own try/except so engine + failure didn't take down dispatch sites). Pin the contract: empty + engine still answers safely on every accessor instead of raising.""" + registry = MediaServerRegistry() + registry.register(ServerSpec( + name='plex', factory=lambda: _FakeClient('plex'), display_name='Plex', + )) + engine = MediaServerEngine( + registry=registry, + clients={}, # No pre-built clients passed. + active_server_resolver=lambda: 'plex', + ) + + # client(name) returns None for every server — engine doesn't crash. + assert engine.client('plex') is None + assert engine.client('jellyfin') is None + # Active-client lookup + is_connected gracefully handle the empty + # case without raising. + assert engine.active_client() is None + assert engine.is_connected() is False + # configured_clients() returns empty dict cleanly. + assert engine.configured_clients() == {} + + +# --------------------------------------------------------------------------- +# Registry encapsulation (engine reaches via public set_instance, +# not the private _instances dict) +# --------------------------------------------------------------------------- + + +def test_engine_uses_registry_set_instance_not_private_attr(): + """Engine constructor with ``clients=`` must hand instances off + via the registry's public ``set_instance(name, client)`` method + rather than reaching into ``registry._instances`` directly. Pin + by giving the registry a ``set_instance`` spy and verifying the + engine calls it.""" + plex = _FakeClient('plex') + registry = MediaServerRegistry() + registry.register(ServerSpec( + name='plex', factory=lambda: _FakeClient('plex'), display_name='Plex', + )) + set_instance_calls = [] + original = registry.set_instance + def _spy(name, client): + set_instance_calls.append((name, client)) + original(name, client) + registry.set_instance = _spy + + MediaServerEngine( + registry=registry, + clients={'plex': plex}, + active_server_resolver=lambda: 'plex', + ) + assert ('plex', plex) in set_instance_calls diff --git a/tests/media_server/test_jellyfin_pinning.py b/tests/media_server/test_jellyfin_pinning.py new file mode 100644 index 00000000..d05f407f --- /dev/null +++ b/tests/media_server/test_jellyfin_pinning.py @@ -0,0 +1,104 @@ +"""Phase A pinning tests for JellyfinClient. + +Pin the OBSERVABLE BEHAVIOR the engine will dispatch through after +Phase B/C. Jellyfin's connection-readiness check is stricter than +Plex (requires base_url + api_key + user_id + music_library_id all +present), and get_all_artists has a side effect of pre-populating +caches. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from core.jellyfin_client import JellyfinClient + + +@pytest.fixture +def jellyfin_client(): + """A bare JellyfinClient with no real connection.""" + client = JellyfinClient.__new__(JellyfinClient) + client.base_url = None + client.api_key = None + client.user_id = None + client.music_library_id = None + client._connection_attempted = False + client._is_connecting = False + # Initialize the various caches the real __init__ sets up + client._album_cache = {} + client._track_cache = {} + client._artist_cache = {} + client._cache_populated = False + client._metadata_only_mode = False + return client + + +# --------------------------------------------------------------------------- +# is_connected +# --------------------------------------------------------------------------- + + +def test_is_connected_returns_false_when_no_credentials(jellyfin_client): + """Pinning: any of base_url / api_key / user_id / music_library_id + being None → False. All four must be set.""" + with patch.object(jellyfin_client, 'ensure_connection', return_value=False): + assert jellyfin_client.is_connected() is False + + +def test_is_connected_returns_true_only_when_all_four_set(jellyfin_client): + """Pinning: requires base_url AND api_key AND user_id AND + music_library_id. Setting only some → still False.""" + jellyfin_client._connection_attempted = True + + # Only some — still False + jellyfin_client.base_url = 'http://j' + jellyfin_client.api_key = 'k' + jellyfin_client.user_id = 'u' + jellyfin_client.music_library_id = None + assert jellyfin_client.is_connected() is False + + # All four — True + jellyfin_client.music_library_id = 'lib' + assert jellyfin_client.is_connected() is True + + +# --------------------------------------------------------------------------- +# get_all_artists / get_all_album_ids +# --------------------------------------------------------------------------- + + +def test_get_all_artists_returns_empty_when_not_connected(jellyfin_client): + with patch.object(jellyfin_client, 'ensure_connection', return_value=False): + assert jellyfin_client.get_all_artists() == [] + + +def test_get_all_artists_returns_empty_when_no_music_library_id(jellyfin_client): + """Pinning: even with ensure_connection True, no music_library_id + selected → empty (don't crash).""" + jellyfin_client.music_library_id = None + with patch.object(jellyfin_client, 'ensure_connection', return_value=True): + assert jellyfin_client.get_all_artists() == [] + + +def test_get_all_album_ids_returns_set(jellyfin_client): + """Pinning: returns a set of string Jellyfin GUID ids. Same shape + semantic as Plex (set of strings) — engine extraction depends on + uniform set type.""" + jellyfin_client._connection_attempted = True + jellyfin_client.base_url = 'http://j' + jellyfin_client.api_key = 'k' + jellyfin_client.user_id = 'u' + jellyfin_client.music_library_id = 'lib' + + fake_response_pages = [ + {'Items': [{'Id': 'guid-1'}, {'Id': 'guid-2'}], 'TotalRecordCount': 2}, + ] + + with patch.object(jellyfin_client, 'ensure_connection', return_value=True), \ + patch.object(jellyfin_client, '_make_request', side_effect=fake_response_pages): + result = jellyfin_client.get_all_album_ids() + + assert isinstance(result, set) + assert result == {'guid-1', 'guid-2'} diff --git a/tests/media_server/test_navidrome_pinning.py b/tests/media_server/test_navidrome_pinning.py new file mode 100644 index 00000000..500d3953 --- /dev/null +++ b/tests/media_server/test_navidrome_pinning.py @@ -0,0 +1,79 @@ +"""Phase A pinning tests for NavidromeClient. + +Pin the OBSERVABLE BEHAVIOR the engine will dispatch through. Auth +shape uses base_url + username + password (no token like Plex). +Both get_all_artists + get_all_album_ids return empty when the +client isn't connected. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from core.navidrome_client import NavidromeClient + + +@pytest.fixture +def nav_client(): + """A bare NavidromeClient with no real connection.""" + client = NavidromeClient.__new__(NavidromeClient) + client.base_url = None + client.username = None + client.password = None + client._connection_attempted = False + client._is_connecting = False + client._artist_cache = {} + client._album_cache = {} + client._track_cache = {} + client._folder_album_ids = None + client._progress_callback = None + client.music_folder_id = None + return client + + +def test_is_connected_returns_false_without_credentials(nav_client): + with patch.object(nav_client, 'ensure_connection', return_value=False): + assert nav_client.is_connected() is False + + +def test_is_connected_returns_true_when_all_three_set(nav_client): + """Pinning: requires base_url AND username AND password. + No token model (vs Plex). Salt is generated per-request.""" + nav_client._connection_attempted = True + nav_client.base_url = 'http://nav' + nav_client.username = 'u' + nav_client.password = 'p' + assert nav_client.is_connected() is True + + +def test_get_all_artists_returns_empty_when_not_connected(nav_client): + with patch.object(nav_client, 'ensure_connection', return_value=False): + assert nav_client.get_all_artists() == [] + + +def test_get_all_album_ids_returns_set(nav_client): + """Pinning: returns a set of string Navidrome ids. Same shape + semantic as Plex/Jellyfin (set of strings) — engine extraction + depends on uniform set type.""" + nav_client._connection_attempted = True + nav_client.base_url = 'http://nav' + nav_client.username = 'u' + nav_client.password = 'p' + + # Navidrome uses paginated getAlbumList2 — first page returns + # 2 albums, second returns empty (terminates loop). + # _make_request unwraps the subsonic-response envelope and + # returns the body. get_all_album_ids reads response.albumList2.album. + page_responses = [ + {'albumList2': {'album': [{'id': 'nav-1'}, {'id': 'nav-2'}]}}, + {'albumList2': {'album': []}}, + ] + + with patch.object(nav_client, 'ensure_connection', return_value=True), \ + patch.object(nav_client, '_make_request', side_effect=page_responses): + result = nav_client.get_all_album_ids() + + assert isinstance(result, set) + assert result == {'nav-1', 'nav-2'} diff --git a/tests/media_server/test_plex_all_libraries.py b/tests/media_server/test_plex_all_libraries.py new file mode 100644 index 00000000..ee4cc823 --- /dev/null +++ b/tests/media_server/test_plex_all_libraries.py @@ -0,0 +1,544 @@ +"""Tests for PlexClient's "All Libraries (combined)" mode. + +When the user picks `ALL_LIBRARIES_SENTINEL` as their saved library +preference, every read method must dispatch through +`server.library.search(libtype=...)` instead of the single-section +`self.music_library.X` path. Pin both modes so future refactors can't +silently break either path. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from core.plex_client import PlexClient, ALL_LIBRARIES_SENTINEL + + +def _make_client(*, all_libraries_mode: bool = False, music_library=None, server=None): + client = PlexClient.__new__(PlexClient) + client.server = server + client.music_library = music_library + client._all_libraries_mode = all_libraries_mode + client._connection_attempted = server is not None + client._is_connecting = False + client._last_connection_check = 0 + client._connection_check_interval = 30 + return client + + +# --------------------------------------------------------------------------- +# Mode flag setup — sentinel handling +# --------------------------------------------------------------------------- + + +def test_set_music_library_by_name_sentinel_enables_all_libraries_mode(monkeypatch): + """Pin: passing the sentinel as library_name flips the client into + all-libraries mode and stores the sentinel as the saved preference.""" + server = MagicMock() + client = _make_client(server=server, music_library=MagicMock()) + + saved_pref = {} + + class _StubDB: + def set_preference(self, key, value): + saved_pref[key] = value + + import database.music_database as db_mod + monkeypatch.setattr(db_mod, 'MusicDatabase', _StubDB) + + result = client.set_music_library_by_name(ALL_LIBRARIES_SENTINEL) + + assert result is True + assert client._all_libraries_mode is True + assert client.music_library is None + assert saved_pref == {'plex_music_library': ALL_LIBRARIES_SENTINEL} + + +def test_set_music_library_by_name_specific_library_disables_all_libraries_mode(monkeypatch): + """Pin: when user switches FROM all-libraries to a specific section, + the mode flag clears.""" + server = MagicMock() + section = MagicMock(type='artist', title='Music') + server.library.sections.return_value = [section] + client = _make_client(server=server, all_libraries_mode=True) + + class _StubDB: + def set_preference(self, key, value): + pass + + import database.music_database as db_mod + monkeypatch.setattr(db_mod, 'MusicDatabase', _StubDB) + + result = client.set_music_library_by_name('Music') + + assert result is True + assert client._all_libraries_mode is False + assert client.music_library is section + + +def test_is_fully_configured_true_in_all_libraries_mode(): + """Pin: all-libraries mode counts as configured even though + music_library is None — UI gating + dispatch site checks rely on + this for the new mode to be functional.""" + server = MagicMock() + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + assert client.is_fully_configured() is True + + +# --------------------------------------------------------------------------- +# Read methods — single-library mode (regression guard) +# --------------------------------------------------------------------------- + + +def test_get_all_artists_uses_section_search_in_single_library_mode(): + """Single-library mode: get_all_artists hits self.music_library.searchArtists() + and never calls server.library.search().""" + server = MagicMock() + section = MagicMock() + section.searchArtists.return_value = [MagicMock(ratingKey=1), MagicMock(ratingKey=2)] + client = _make_client(server=server, music_library=section, all_libraries_mode=False) + + result = client.get_all_artists() + + section.searchArtists.assert_called_once() + server.library.search.assert_not_called() + assert len(result) == 2 + + +def test_get_all_album_ids_uses_section_albums_in_single_library_mode(): + server = MagicMock() + section = MagicMock() + section.albums.return_value = [MagicMock(ratingKey=10), MagicMock(ratingKey=20)] + client = _make_client(server=server, music_library=section, all_libraries_mode=False) + + ids = client.get_all_album_ids() + + section.albums.assert_called_once() + server.library.search.assert_not_called() + assert ids == {'10', '20'} + + +# --------------------------------------------------------------------------- +# Read methods — all-libraries mode (the new path) +# --------------------------------------------------------------------------- + + +def test_get_all_artists_uses_server_search_in_all_libraries_mode(): + """All-libraries mode: get_all_artists dispatches through + server.library.search(libtype='artist') — server-wide aggregation + via Plex's API, no per-section iteration.""" + server = MagicMock() + server.library.search.return_value = [MagicMock(ratingKey=1), MagicMock(ratingKey=2), MagicMock(ratingKey=3)] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + result = client.get_all_artists() + + server.library.search.assert_called_once_with(libtype='artist') + assert len(result) == 3 + + +def test_get_all_album_ids_uses_server_search_in_all_libraries_mode(): + server = MagicMock() + server.library.search.return_value = [ + MagicMock(ratingKey=10), MagicMock(ratingKey=20), MagicMock(ratingKey=30), + ] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + ids = client.get_all_album_ids() + + server.library.search.assert_called_once_with(libtype='album') + assert ids == {'10', '20', '30'} + + +def test_get_library_stats_unions_across_sections_in_all_libraries_mode(): + """All-libraries stats sum totals across every music section via + server-wide search calls. Artists / albums use distinct names so + dedup doesn't collapse them — separate test pins the dedup path.""" + server = MagicMock() + distinct_artists = [_fake_artist(f'Artist {i}', rating_key=str(i), leaf_count=i + 1) for i in range(5)] + distinct_albums = [_fake_album(f'Album {i}', parent=f'Artist {i}', rating_key=str(100 + i), leaf_count=i + 1) for i in range(12)] + distinct_tracks = [MagicMock(ratingKey=str(1000 + i)) for i in range(87)] + server.library.search.side_effect = [ + distinct_artists, + distinct_albums, + distinct_tracks, + ] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + stats = client.get_library_stats() + + assert stats == {'artists': 5, 'albums': 12, 'tracks': 87} + assert server.library.search.call_count == 3 + + +# --------------------------------------------------------------------------- +# Trigger / scanning — multi-section fan-out +# --------------------------------------------------------------------------- + + +def test_trigger_library_scan_fans_out_in_all_libraries_mode(): + """Pin: in all-libraries mode, trigger_library_scan calls update() + on every music section, not just the named one.""" + section_a = MagicMock(type='artist', title='Music A') + section_b = MagicMock(type='artist', title='Music B') + section_other = MagicMock(type='movie', title='Movies') + server = MagicMock() + server.library.sections.return_value = [section_a, section_b, section_other] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + # Bypass ensure_connection's internal logic. + client.ensure_connection = lambda: True + + result = client.trigger_library_scan('ignored-name') + + section_a.update.assert_called_once() + section_b.update.assert_called_once() + section_other.update.assert_not_called() + assert result is True + + +def test_is_library_scanning_returns_true_when_any_section_refreshing(): + """Pin: in all-libraries mode, is_library_scanning returns True + if any music section has refreshing=True.""" + section_a = MagicMock(type='artist', title='A', refreshing=False) + section_b = MagicMock(type='artist', title='B', refreshing=True) # actively scanning + server = MagicMock() + server.library.sections.return_value = [section_a, section_b] + server.activities.return_value = [] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + client.ensure_connection = lambda: True + + assert client.is_library_scanning() is True + + +# --------------------------------------------------------------------------- +# get_available_music_libraries — synthetic sentinel entry +# --------------------------------------------------------------------------- + + +def test_available_music_libraries_prepends_sentinel_when_multiple_libraries(): + """Pin: when more than one music library exists, prepend "All + Libraries (combined)" with the sentinel as its key. Single-library + users don't get the extra option.""" + server = MagicMock() + sec_a = MagicMock(type='artist', title='Music', key=1) + sec_b = MagicMock(type='artist', title='Audiobook Library', key=2) + server.library.sections.return_value = [sec_a, sec_b] + client = _make_client(server=server, music_library=sec_a, all_libraries_mode=False) + client.ensure_connection = lambda: True + + libs = client.get_available_music_libraries() + + # Synthetic entry first; ``value`` carries the sentinel so the + # frontend has a canonical identifier to POST back to the + # select-library endpoint. + assert libs[0]['title'] == 'All Libraries (combined)' + assert libs[0]['key'] == ALL_LIBRARIES_SENTINEL + assert libs[0]['value'] == ALL_LIBRARIES_SENTINEL + assert any(l['title'] == 'Music' and l['value'] == 'Music' for l in libs) + assert any(l['title'] == 'Audiobook Library' and l['value'] == 'Audiobook Library' for l in libs) + + +def test_available_music_libraries_omits_sentinel_when_only_one_library(): + """Single-library users don't need the option — keep the dropdown + clean.""" + server = MagicMock() + sec_a = MagicMock(type='artist', title='Music', key=1) + server.library.sections.return_value = [sec_a] + client = _make_client(server=server, music_library=sec_a, all_libraries_mode=False) + client.ensure_connection = lambda: True + + libs = client.get_available_music_libraries() + + assert all(l['key'] != ALL_LIBRARIES_SENTINEL for l in libs) + assert len(libs) == 1 + + +# --------------------------------------------------------------------------- +# Defensive — _can_query / _all_artists / etc gracefully handle no-config +# --------------------------------------------------------------------------- + + +def test_can_query_false_when_no_section_and_not_all_libraries(): + server = MagicMock() + client = _make_client(server=server, music_library=None, all_libraries_mode=False) + assert client._can_query() is False + + +def test_can_query_false_when_no_server(): + client = _make_client(server=None, music_library=None, all_libraries_mode=True) + assert client._can_query() is False + + +def test_all_artists_returns_empty_when_not_configured(): + client = _make_client(server=None, all_libraries_mode=False) + assert client._all_artists() == [] + + +# --------------------------------------------------------------------------- +# Helper methods for downstream callers (DatabaseUpdateWorker, web_server) +# --------------------------------------------------------------------------- + + +def test_get_recently_added_albums_unions_across_sections_in_all_libraries_mode(): + """Pin: ``get_recently_added_albums`` iterates every music section + in all-libraries mode and concats the recentlyAdded list. Pre-fix + DatabaseUpdateWorker reached ``music_library.recentlyAdded()`` + directly which crashed when music_library is None.""" + section_a = MagicMock(type='artist', title='A') + section_a.recentlyAdded.return_value = [MagicMock(ratingKey=1), MagicMock(ratingKey=2)] + section_b = MagicMock(type='artist', title='B') + section_b.recentlyAdded.return_value = [MagicMock(ratingKey=3)] + server = MagicMock() + server.library.sections.return_value = [section_a, section_b] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + client.ensure_connection = lambda: True + + result = client.get_recently_added_albums(maxresults=100) + + section_a.recentlyAdded.assert_called_once_with(libtype='album', maxresults=100) + section_b.recentlyAdded.assert_called_once_with(libtype='album', maxresults=100) + assert len(result) == 3 + + +def test_get_recently_added_albums_uses_section_in_single_library_mode(): + section = MagicMock() + section.recentlyAdded.return_value = [MagicMock(ratingKey=1)] + server = MagicMock() + client = _make_client(server=server, music_library=section, all_libraries_mode=False) + client.ensure_connection = lambda: True + + result = client.get_recently_added_albums(maxresults=50) + + section.recentlyAdded.assert_called_once_with(libtype='album', maxresults=50) + assert len(result) == 1 + + +def test_get_recently_added_albums_libtype_none_skips_filter(): + """Pin: ``libtype=None`` skips the libtype kwarg entirely so + callers can fetch mixed types when album-only returned nothing.""" + section = MagicMock() + section.recentlyAdded.return_value = [] + server = MagicMock() + client = _make_client(server=server, music_library=section, all_libraries_mode=False) + client.ensure_connection = lambda: True + + client.get_recently_added_albums(maxresults=50, libtype=None) + + section.recentlyAdded.assert_called_once_with(maxresults=50) + + +def test_get_recently_updated_albums_uses_search_dispatch(): + """Pin: routes through ``_search_general`` so single-section vs + all-libraries mode is handled by the helper.""" + server = MagicMock() + server.library.search.return_value = [MagicMock(ratingKey=1), MagicMock(ratingKey=2)] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + client.ensure_connection = lambda: True + + result = client.get_recently_updated_albums(limit=200) + + server.library.search.assert_called_once_with(sort='updatedAt:desc', libtype='album', limit=200) + assert len(result) == 2 + + +def test_get_music_library_locations_unions_across_sections_in_all_libraries_mode(): + """Pin: ``get_music_library_locations`` returns every folder root + configured across every music section. web_server.py uses this for + file-path resolution — pre-fix it reached ``music_library.locations`` + which is None in all-libraries mode.""" + section_a = MagicMock(type='artist', title='A') + section_a.locations = ['/data/userOne/Artists', '/data/userOne/More'] + section_b = MagicMock(type='artist', title='B') + section_b.locations = ['/data/userTwo/Artists'] + server = MagicMock() + server.library.sections.return_value = [section_a, section_b] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + client.ensure_connection = lambda: True + + locations = client.get_music_library_locations() + + assert '/data/userOne/Artists' in locations + assert '/data/userOne/More' in locations + assert '/data/userTwo/Artists' in locations + assert len(locations) == 3 + + +def test_get_music_library_locations_dedupes_overlapping_paths(): + """Pin: same root listed in multiple sections returns once.""" + section_a = MagicMock(type='artist', title='A') + section_a.locations = ['/data/shared'] + section_b = MagicMock(type='artist', title='B') + section_b.locations = ['/data/shared', '/data/userTwo'] + server = MagicMock() + server.library.sections.return_value = [section_a, section_b] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + client.ensure_connection = lambda: True + + locations = client.get_music_library_locations() + + assert locations.count('/data/shared') == 1 + assert '/data/userTwo' in locations + + +# --------------------------------------------------------------------------- +# Cross-section dedup (only active in all-libraries mode) +# --------------------------------------------------------------------------- + + +def _fake_artist(name, rating_key, leaf_count): + a = MagicMock() + a.title = name + a.ratingKey = rating_key + a.leafCount = leaf_count + return a + + +def _fake_album(title, parent, rating_key, leaf_count): + a = MagicMock() + a.title = title + a.parentTitle = parent + a.ratingKey = rating_key + a.leafCount = leaf_count + return a + + +def test_dedupe_artists_keeps_canonical_with_higher_track_count(): + """Pin: same-name artists across sections collapse to one — the + one with the higher leafCount wins. Plex Home users with overlapping + music tastes (both have Drake) shouldn't see "Drake" twice in + SoulSync's library.""" + server = MagicMock() + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + drake_a = _fake_artist('Drake', rating_key='1', leaf_count=12) + drake_b = _fake_artist('Drake', rating_key='2', leaf_count=87) # canonical + kendrick = _fake_artist('Kendrick Lamar', rating_key='3', leaf_count=40) + + deduped = client._dedupe_artists([drake_a, drake_b, kendrick]) + + names = sorted(a.title for a in deduped) + assert names == ['Drake', 'Kendrick Lamar'] + drake_picked = next(a for a in deduped if a.title == 'Drake') + assert drake_picked.ratingKey == '2' # higher leafCount wins + + +def test_dedupe_artists_case_insensitive_match(): + """Pin: dedup matches lowercased name so "Drake" + "drake" + "DRAKE" + collapse together.""" + server = MagicMock() + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + a1 = _fake_artist('Drake', rating_key='1', leaf_count=10) + a2 = _fake_artist('drake', rating_key='2', leaf_count=20) + a3 = _fake_artist('DRAKE', rating_key='3', leaf_count=5) + + deduped = client._dedupe_artists([a1, a2, a3]) + + assert len(deduped) == 1 + assert deduped[0].ratingKey == '2' # canonical = highest count + + +def test_dedupe_artists_noop_in_single_library_mode(): + """Pin: dedup is bypassed entirely in single-library mode — the + input list comes back unchanged. Single-section users get zero + behavior change from the dedup logic.""" + server = MagicMock() + section = MagicMock() + client = _make_client(server=server, music_library=section, all_libraries_mode=False) + + drake_a = _fake_artist('Drake', rating_key='1', leaf_count=12) + drake_b = _fake_artist('Drake', rating_key='2', leaf_count=87) + + result = client._dedupe_artists([drake_a, drake_b]) + + assert len(result) == 2 + assert result == [drake_a, drake_b] + + +def test_dedupe_albums_groups_by_artist_and_title(): + """Pin: album dedup keys on (artist, title) so two artists with + same album title (e.g. self-titled) stay separate.""" + server = MagicMock() + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + drake_self_a = _fake_album('Drake', parent='Drake', rating_key='1', leaf_count=15) + drake_self_b = _fake_album('Drake', parent='Drake', rating_key='2', leaf_count=15) + weeknd_drake = _fake_album('Drake', parent='The Weeknd', rating_key='3', leaf_count=8) # different artist + + deduped = client._dedupe_albums([drake_self_a, drake_self_b, weeknd_drake]) + + assert len(deduped) == 2 + artists = sorted(a.parentTitle for a in deduped) + assert artists == ['Drake', 'The Weeknd'] + + +def test_get_all_artists_dedupes_in_all_libraries_mode(): + """Pin: ``get_all_artists`` (the public listing) returns deduped + list in all-libraries mode.""" + server = MagicMock() + drake_a = _fake_artist('Drake', rating_key='1', leaf_count=12) + drake_b = _fake_artist('Drake', rating_key='2', leaf_count=87) + server.library.search.return_value = [drake_a, drake_b] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + result = client.get_all_artists() + + assert len(result) == 1 + assert result[0].ratingKey == '2' + + +def test_get_all_artist_ids_does_NOT_dedupe_critical_for_removal_detection(): + """CRITICAL pin: ``get_all_artist_ids`` returns the RAW ratingKey + set, even in all-libraries mode. Removal detection compares this + set against DB-linked ratingKeys to decide what's been removed — + deduping here would falsely report non-canonical ratingKeys as + "removed" and prune library tracks pointing at them.""" + server = MagicMock() + drake_a = _fake_artist('Drake', rating_key='1', leaf_count=12) + drake_b = _fake_artist('Drake', rating_key='2', leaf_count=87) + server.library.search.return_value = [drake_a, drake_b] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + ids = client.get_all_artist_ids() + + # Both ratingKeys returned, NOT deduped. + assert ids == {'1', '2'} + + +def test_get_all_album_ids_does_NOT_dedupe_critical_for_removal_detection(): + """Same critical pin for albums.""" + server = MagicMock() + alb_a = _fake_album('Take Care', parent='Drake', rating_key='10', leaf_count=15) + alb_b = _fake_album('Take Care', parent='Drake', rating_key='20', leaf_count=15) + server.library.search.return_value = [alb_a, alb_b] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + ids = client.get_all_album_ids() + + assert ids == {'10', '20'} + + +def test_get_library_stats_uses_deduped_counts_for_artists_albums(): + """Pin: stats reflect the deduped counts users see in the library + list. Tracks stay raw — same track in two sections is two files.""" + server = MagicMock() + drake_a = _fake_artist('Drake', rating_key='1', leaf_count=12) + drake_b = _fake_artist('Drake', rating_key='2', leaf_count=87) + kendrick = _fake_artist('Kendrick Lamar', rating_key='3', leaf_count=40) + alb_a = _fake_album('Take Care', parent='Drake', rating_key='10', leaf_count=15) + alb_b = _fake_album('Take Care', parent='Drake', rating_key='20', leaf_count=15) + track1 = MagicMock(ratingKey=100) + track2 = MagicMock(ratingKey=101) + track3 = MagicMock(ratingKey=102) + server.library.search.side_effect = [ + [drake_a, drake_b, kendrick], # artist call → 3 raw → 2 deduped + [alb_a, alb_b], # album call → 2 raw → 1 deduped + [track1, track2, track3], # track call → 3 raw, no dedup + ] + client = _make_client(server=server, all_libraries_mode=True, music_library=None) + + stats = client.get_library_stats() + + assert stats == {'artists': 2, 'albums': 1, 'tracks': 3} diff --git a/tests/media_server/test_plex_pinning.py b/tests/media_server/test_plex_pinning.py new file mode 100644 index 00000000..9e3a3ba7 --- /dev/null +++ b/tests/media_server/test_plex_pinning.py @@ -0,0 +1,114 @@ +"""Phase A pinning tests for PlexClient. + +Pin the OBSERVABLE BEHAVIOR the engine will dispatch through after +Phase B/C. The web_server.py dispatch sites + DatabaseUpdateWorker +read these methods generically — they must keep their current shape +through the engine refactor. + +Plex's surface is wider than the contract requires (additional +playlist / metadata writeback methods), but pinning ALL of them +turns into ~30 tests. Focused on the methods the dispatch sites +actually call: is_connected, is_fully_configured, ensure_connection, +get_all_artists, get_all_album_ids, search_tracks, trigger_library_scan, +is_library_scanning, get_library_stats. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from core.plex_client import PlexClient + + +@pytest.fixture +def plex_client(): + """A bare PlexClient with no real connection. Tests that need + a connected state set client.server = MagicMock() directly.""" + client = PlexClient.__new__(PlexClient) + client.server = None + client.music_library = None + client._all_libraries_mode = False + client._connection_attempted = False + client._is_connecting = False + client._last_connection_check = 0 + client._connection_check_interval = 30 + return client + + +# --------------------------------------------------------------------------- +# is_connected +# --------------------------------------------------------------------------- + + +def test_is_connected_returns_false_when_no_server(plex_client): + """Pinning: no server object → False. Dashboard status indicators + + endpoint guards depend on this.""" + plex_client.server = None + # Patch ensure_connection to avoid network call + with patch.object(plex_client, 'ensure_connection', return_value=False): + assert plex_client.is_connected() is False + + +def test_is_connected_returns_true_when_server_present(plex_client): + """Pinning: a non-None server object → True (even if music_library + is unset). is_fully_configured() is the stricter check.""" + plex_client.server = MagicMock() + plex_client._connection_attempted = True + assert plex_client.is_connected() is True + + +def test_is_fully_configured_requires_server_and_music_library(plex_client): + """Pinning: is_fully_configured == True only when BOTH server AND + music_library are set. Used by playlist-sync gating.""" + plex_client.server = MagicMock() + plex_client.music_library = None + assert plex_client.is_fully_configured() is False + plex_client.music_library = MagicMock() + assert plex_client.is_fully_configured() is True + + +# --------------------------------------------------------------------------- +# get_all_artists / get_all_album_ids +# --------------------------------------------------------------------------- + + +def test_get_all_artists_returns_empty_when_not_connected(plex_client): + """Pinning: ensure_connection returning False → empty list (not + exception). Caller iterates the result, never raising.""" + with patch.object(plex_client, 'ensure_connection', return_value=False): + assert plex_client.get_all_artists() == [] + + +def test_get_all_artists_iterates_searchartists_when_connected(plex_client): + """Pinning: when connected, get_all_artists calls + music_library.searchArtists() and returns the result list.""" + fake_artists = [MagicMock(title='A'), MagicMock(title='B')] + plex_client.server = MagicMock() + plex_client.music_library = MagicMock() + plex_client.music_library.searchArtists.return_value = fake_artists + + with patch.object(plex_client, 'ensure_connection', return_value=True): + result = plex_client.get_all_artists() + + assert result == fake_artists + + +def test_get_all_album_ids_returns_set_of_string_ratingkeys(plex_client): + """Pinning: returns a set (not a list) of string ratingKey values. + DatabaseUpdateWorker uses set membership for diff-detection. + NOTE: Plex stores ratingKey as int but the method coerces to str + so the set semantics match other servers (Jellyfin GUIDs, Navidrome + string ids). Engine refactor must preserve.""" + fake_albums = [MagicMock(ratingKey=1), MagicMock(ratingKey=2), + MagicMock(ratingKey=3)] + plex_client.server = MagicMock() + plex_client.music_library = MagicMock() + plex_client.music_library.albums.return_value = fake_albums + + with patch.object(plex_client, 'ensure_connection', return_value=True): + result = plex_client.get_all_album_ids() + + assert isinstance(result, set) + assert result == {'1', '2', '3'} diff --git a/tests/media_server/test_soulsync_pinning.py b/tests/media_server/test_soulsync_pinning.py new file mode 100644 index 00000000..d419c829 --- /dev/null +++ b/tests/media_server/test_soulsync_pinning.py @@ -0,0 +1,90 @@ +"""Phase A pinning tests for SoulSyncClient (standalone library mode). + +Pin the OBSERVABLE BEHAVIOR the engine will dispatch through. +SoulSync standalone is the structurally-different one — no auth / +no API / no library scan. is_connected just checks `transfer_path` +is a directory. Filesystem walk happens via _get_cached_scan(). +""" + +from __future__ import annotations + +import os +from unittest.mock import patch + +import pytest + +from core.soulsync_client import SoulSyncClient + + +@pytest.fixture +def ss_client(tmp_path): + """A bare SoulSyncClient pointed at a temp directory.""" + client = SoulSyncClient.__new__(SoulSyncClient) + client._transfer_path = str(tmp_path) + client._progress_callback = None + client._cache = None + client._cache_time = 0 + client._cache_ttl = 300 + client._last_scan_time = None + return client + + +# --------------------------------------------------------------------------- +# is_connected / ensure_connection +# --------------------------------------------------------------------------- + + +def test_is_connected_true_when_transfer_path_exists(ss_client, tmp_path): + """Pinning: filesystem-based connectivity. is_connected just + checks os.path.isdir(transfer_path).""" + assert ss_client.is_connected() is True # tmp_path exists + + +def test_is_connected_false_when_transfer_path_missing(ss_client): + """Pinning: missing transfer_path → False. The standalone case + where the user hasn't configured a folder yet.""" + ss_client._transfer_path = '/nonexistent/path/that/does/not/exist' + assert ss_client.is_connected() is False + + +def test_ensure_connection_reloads_config_then_checks_path(ss_client, tmp_path): + """Pinning: ensure_connection re-reads config (so the user + changing the transfer_path takes effect without a process + restart) and then checks the path. Returns True iff the path + is a directory.""" + with patch.object(ss_client, '_reload_config') as fake_reload: + result = ss_client.ensure_connection() + + fake_reload.assert_called_once() + assert result is True + + +# --------------------------------------------------------------------------- +# get_all_artists / get_all_album_ids +# --------------------------------------------------------------------------- + + +def test_get_all_artists_uses_cached_scan(ss_client): + """Pinning: get_all_artists delegates to _get_cached_scan which + enforces the 5-min TTL. Engine extraction must preserve.""" + fake_artists = [object(), object(), object()] + with patch.object(ss_client, '_get_cached_scan', return_value=fake_artists): + result = ss_client.get_all_artists() + assert result == fake_artists + + +def test_get_all_album_ids_returns_set(ss_client): + """Pinning: returns a set of MD5-hashed string ids. Same uniform + set semantics as Plex/Jellyfin/Navidrome.""" + fake_album = type('FakeAlbum', (), {'ratingKey': 'hash-1'})() + fake_album2 = type('FakeAlbum', (), {'ratingKey': 'hash-2'})() + # Real SoulSyncArtist exposes albums() method (not _albums attr). + fake_artist = type('FakeArtist', (), { + 'albums': lambda self: [fake_album, fake_album2], + })() + + with patch.object(ss_client, '_get_cached_scan', return_value=[fake_artist]): + result = ss_client.get_all_album_ids() + + assert isinstance(result, set) + assert result == {'hash-1', 'hash-2'} diff --git a/tests/metadata/test_album_mbid_cache.py b/tests/metadata/test_album_mbid_cache.py new file mode 100644 index 00000000..ba17f9fa --- /dev/null +++ b/tests/metadata/test_album_mbid_cache.py @@ -0,0 +1,207 @@ +"""Tests for ``core/metadata/album_mbid_cache.py``. + +The persistent MBID cache is the root-cause fix for "tracks of one +album get different MUSICBRAINZ_ALBUMID tags after the in-memory cache +evicts or after a server restart." Strict additive design: every +public function degrades to None / no-op on any database error so the +existing in-memory cache + MusicBrainz lookup remains the +authoritative fallback. + +These tests pin: round-trip lookup/record, idempotent re-record, +clear_all behavior, defensive None-on-empty-input, and the graceful +degradation path when the database accessor fails. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from core.metadata import album_mbid_cache +from database.music_database import get_database + + +@pytest.fixture(autouse=True) +def _wipe_cache(): + """Each test starts with a clean persistent cache so rows from + earlier tests don't leak. Wipe AFTER too so other test files + aren't affected.""" + album_mbid_cache.clear_all() + yield + album_mbid_cache.clear_all() + + +# --------------------------------------------------------------------------- +# Happy path +# --------------------------------------------------------------------------- + + +def test_lookup_returns_none_for_missing_key() -> None: + assert album_mbid_cache.lookup('nonexistent', 'nobody') is None + + +def test_record_then_lookup_roundtrip() -> None: + assert album_mbid_cache.record('gnx', 'kendrick lamar', 'release-mbid-abc-123') is True + assert album_mbid_cache.lookup('gnx', 'kendrick lamar') == 'release-mbid-abc-123' + + +def test_record_is_idempotent() -> None: + """Re-recording the same key with the same MBID succeeds and the + lookup still returns the value. Schema uses INSERT OR REPLACE.""" + assert album_mbid_cache.record('gnx', 'kendrick lamar', 'mbid-1') is True + assert album_mbid_cache.record('gnx', 'kendrick lamar', 'mbid-1') is True + assert album_mbid_cache.lookup('gnx', 'kendrick lamar') == 'mbid-1' + + +def test_record_overwrites_with_new_mbid() -> None: + """If the same (album, artist) pair gets re-recorded with a + different MBID, the new value wins (last-write-wins). Surfaces + the case where MusicBrainz reorganized a release id.""" + album_mbid_cache.record('gnx', 'kendrick lamar', 'old-mbid') + album_mbid_cache.record('gnx', 'kendrick lamar', 'new-mbid') + assert album_mbid_cache.lookup('gnx', 'kendrick lamar') == 'new-mbid' + + +def test_clear_all_wipes_all_entries() -> None: + album_mbid_cache.record('a', 'artist1', 'mbid-1') + album_mbid_cache.record('b', 'artist2', 'mbid-2') + assert album_mbid_cache.lookup('a', 'artist1') == 'mbid-1' + assert album_mbid_cache.lookup('b', 'artist2') == 'mbid-2' + + assert album_mbid_cache.clear_all() is True + assert album_mbid_cache.lookup('a', 'artist1') is None + assert album_mbid_cache.lookup('b', 'artist2') is None + + +def test_lookup_preserves_album_artist_independence() -> None: + """Same album name across different artists must NOT collide. + Compilation albums titled 'Greatest Hits' would otherwise clobber + each other across artists.""" + album_mbid_cache.record('greatest hits', 'queen', 'queen-mbid') + album_mbid_cache.record('greatest hits', 'eminem', 'eminem-mbid') + assert album_mbid_cache.lookup('greatest hits', 'queen') == 'queen-mbid' + assert album_mbid_cache.lookup('greatest hits', 'eminem') == 'eminem-mbid' + + +# --------------------------------------------------------------------------- +# Defensive paths — must NEVER raise to caller +# --------------------------------------------------------------------------- + + +def test_lookup_returns_none_for_empty_album_key() -> None: + assert album_mbid_cache.lookup('', 'kendrick') is None + + +def test_lookup_returns_none_for_empty_artist_key() -> None: + assert album_mbid_cache.lookup('gnx', '') is None + + +def test_lookup_returns_none_for_none_inputs() -> None: + assert album_mbid_cache.lookup(None, 'kendrick') is None # type: ignore[arg-type] + assert album_mbid_cache.lookup('gnx', None) is None # type: ignore[arg-type] + + +def test_record_returns_false_for_empty_inputs() -> None: + assert album_mbid_cache.record('', 'artist', 'mbid') is False + assert album_mbid_cache.record('album', '', 'mbid') is False + assert album_mbid_cache.record('album', 'artist', '') is False + + +def test_lookup_degrades_to_none_when_db_unavailable() -> None: + """Critical defensive path: if `_get_database()` returns None + (DB module failed to load, accessor raised, etc), lookup MUST + return None — NOT raise. This is what keeps the enrichment path + working when this layer breaks.""" + with patch.object(album_mbid_cache, '_get_database', return_value=None): + assert album_mbid_cache.lookup('gnx', 'kendrick') is None + + +def test_record_degrades_to_false_when_db_unavailable() -> None: + """Same defensive contract for record.""" + with patch.object(album_mbid_cache, '_get_database', return_value=None): + assert album_mbid_cache.record('gnx', 'kendrick', 'mbid') is False + + +def test_lookup_degrades_to_none_when_query_raises() -> None: + """If the underlying SQL execute throws (locked DB, schema drift, + etc), lookup must catch it and return None. No exception escapes.""" + + class _ExplodingConn: + def cursor(self): + raise RuntimeError("simulated DB explosion") + + def close(self): + pass + + class _StubDB: + def _get_connection(self): + return _ExplodingConn() + + with patch.object(album_mbid_cache, '_get_database', return_value=_StubDB()): + assert album_mbid_cache.lookup('gnx', 'kendrick') is None + + +def test_record_degrades_to_false_when_commit_raises() -> None: + """If commit fails, record returns False — caller (enrichment path) + just doesn't get the persistent benefit, but downloads continue.""" + + class _BadCommitConn: + def cursor(self): + class _C: + def execute(self, *a, **kw): + pass + return _C() + + def commit(self): + raise RuntimeError("commit failed") + + def close(self): + pass + + class _StubDB: + def _get_connection(self): + return _BadCommitConn() + + with patch.object(album_mbid_cache, '_get_database', return_value=_StubDB()): + assert album_mbid_cache.record('gnx', 'kendrick', 'mbid') is False + + +# --------------------------------------------------------------------------- +# Schema sanity +# --------------------------------------------------------------------------- + + +def test_table_exists_after_database_init() -> None: + """The migration in `database/music_database.py` should create the + `mb_album_release_cache` table on database init.""" + db = get_database() + conn = db._get_connection() + try: + cur = conn.cursor() + cur.execute( + "SELECT name FROM sqlite_master WHERE type='table' " + "AND name='mb_album_release_cache'" + ) + row = cur.fetchone() + assert row is not None + finally: + conn.close() + + +def test_release_mbid_index_exists() -> None: + """Reverse-lookup index (find all albums for a given MBID) helps + future debug tooling. Pin its existence so a future migration + refactor doesn't quietly drop it.""" + db = get_database() + conn = db._get_connection() + try: + cur = conn.cursor() + cur.execute( + "SELECT name FROM sqlite_master WHERE type='index' " + "AND name='idx_mb_album_release_mbid'" + ) + row = cur.fetchone() + assert row is not None + finally: + conn.close() diff --git a/tests/metadata/test_build_album_info_typed_path.py b/tests/metadata/test_build_album_info_typed_path.py new file mode 100644 index 00000000..70c69d81 --- /dev/null +++ b/tests/metadata/test_build_album_info_typed_path.py @@ -0,0 +1,203 @@ +"""Pin the typed-path migration of `_build_album_info`. + +`core/metadata/album_tracks.py:_build_album_info` historically did +duck-typed extraction with fallback chains. Step 2 of the typed +metadata migration routes it through `Album.from__dict()` +when the caller provides a recognized `source` argument; legacy +duck-typing kicks in as fallback. + +These tests pin: +- Typed path is taken when `source` is a known provider. +- Output matches the legacy path on the fields the legacy code + produced (the real concern — downstream consumers must not break). +- Legacy path still runs unchanged when `source` is empty/unknown, + or when the typed converter raises. +- Caller-provided `album_id` / `album_name` / `artist_name` + fallbacks apply on the typed path the same way they did on the + legacy path. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from core.metadata import album_tracks + + +# --------------------------------------------------------------------------- +# Typed path is exercised when source is recognized +# --------------------------------------------------------------------------- + + +SAMPLE_SPOTIFY_ALBUM = { + 'id': 'sp123', + 'name': 'GNX', + 'artists': [{'id': 'kdot', 'name': 'Kendrick Lamar'}], + 'release_date': '2024-11-22', + 'total_tracks': 12, + 'album_type': 'album', + 'images': [ + {'url': 'https://i.scdn.co/640.jpg', 'height': 640, 'width': 640}, + {'url': 'https://i.scdn.co/300.jpg', 'height': 300, 'width': 300}, + ], + 'genres': ['hip hop'], + 'label': 'pgLang', +} + + +def test_typed_path_used_for_known_source(): + info = album_tracks._build_album_info( + SAMPLE_SPOTIFY_ALBUM, album_id='sp123', + album_name='', artist_name='', source='spotify', + ) + # Typed converter populates `source` field — legacy path doesn't. + assert info['source'] == 'spotify' + # Typed converter exposes label / genres — legacy doesn't. + assert info['label'] == 'pgLang' + assert info['genres'] == ['hip hop'] + # Core fields match expected values. + assert info['id'] == 'sp123' + assert info['name'] == 'GNX' + assert info['artist'] == 'Kendrick Lamar' + assert info['artist_name'] == 'Kendrick Lamar' + assert info['artist_id'] == 'kdot' + assert info['release_date'] == '2024-11-22' + assert info['album_type'] == 'album' + assert info['total_tracks'] == 12 + + +def test_typed_path_preserves_full_images_list(): + """Legacy code passed the source's full multi-resolution images + list through verbatim. Some downstream consumers iterate it to + pick a different size. Typed path must preserve this.""" + info = album_tracks._build_album_info( + SAMPLE_SPOTIFY_ALBUM, album_id='sp123', source='spotify', + ) + assert len(info['images']) == 2 + assert info['images'][0]['url'] == 'https://i.scdn.co/640.jpg' + assert info['images'][1]['url'] == 'https://i.scdn.co/300.jpg' + + +def test_typed_path_applies_caller_fallbacks_for_missing_fields(): + """When the raw response lacks id/name/artist, the legacy code + used the caller-provided defaults. Typed path must do the same.""" + minimal = {'name': 'X'} # no id, no artists + info = album_tracks._build_album_info( + minimal, album_id='fallback_id', album_name='fallback_name', + artist_name='Fallback Artist', source='spotify', + ) + assert info['id'] == 'fallback_id' + assert info['artist'] == 'Fallback Artist' + assert info['artist_name'] == 'Fallback Artist' + # artists list reflects the caller-provided name (id may be None on + # the typed path since no id was discoverable in raw data — legacy + # used '' but no consumer differentiates None vs '' here). + assert info['artists'][0]['name'] == 'Fallback Artist' + + +# --------------------------------------------------------------------------- +# Legacy path still kicks in for unknown / missing source +# --------------------------------------------------------------------------- + + +def test_legacy_path_used_when_no_source_provided(): + """No source → legacy duck-typed extraction. Backward-compat for + every existing caller that hasn't been migrated yet.""" + info = album_tracks._build_album_info( + SAMPLE_SPOTIFY_ALBUM, album_id='sp123', + ) + # Legacy path doesn't populate `source` field. + assert 'source' not in info or not info.get('source') + # Core fields still resolved correctly via duck-typing. + assert info['id'] == 'sp123' + assert info['name'] == 'GNX' + assert info['artist'] == 'Kendrick Lamar' + + +def test_legacy_path_used_for_unknown_source(): + """Source that doesn't match any registered converter → legacy.""" + info = album_tracks._build_album_info( + SAMPLE_SPOTIFY_ALBUM, album_id='sp123', source='made_up_provider', + ) + assert 'source' not in info or not info.get('source') + + +def test_legacy_path_used_when_album_data_not_dict(): + """Defensive: if the raw input isn't a dict (rare but possible — + some clients return objects), typed path can't apply.""" + class _Obj: + id = 'x' + name = 'Y' + info = album_tracks._build_album_info(_Obj(), album_id='x', source='spotify') + # Falls through to legacy path which uses _extract_lookup_value + # with getattr fallbacks. Result still has core fields. + assert info['id'] == 'x' + assert info['name'] == 'Y' + + +def test_legacy_path_used_when_typed_converter_raises(): + """If the typed converter throws, fall back to legacy. A converter + bug must NOT break album resolution.""" + bad_input = {'id': 'sp123', 'name': 'GNX'} + + def _exploding_converter(_): + raise RuntimeError('simulated converter bug') + + with patch.dict(album_tracks._TYPED_ALBUM_CONVERTERS, + {'spotify': _exploding_converter}): + info = album_tracks._build_album_info( + bad_input, album_id='sp123', source='spotify', + ) + # Legacy path resolved core fields successfully. + assert info['id'] == 'sp123' + assert info['name'] == 'GNX' + # Source field NOT set (legacy path doesn't add it). + assert 'source' not in info or not info.get('source') + + +# --------------------------------------------------------------------------- +# Cross-provider: typed path works for every registered source +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('source,raw,expected_name,expected_artist', [ + ('spotify', SAMPLE_SPOTIFY_ALBUM, 'GNX', 'Kendrick Lamar'), + ('itunes', { + 'collectionId': 1, 'collectionName': 'GNX', + 'artistName': 'Kendrick Lamar', 'trackCount': 12, + }, 'GNX', 'Kendrick Lamar'), + ('deezer', { + 'id': 1, 'title': 'GNX', + 'artist': {'id': 2, 'name': 'Kendrick Lamar'}, + 'nb_tracks': 12, + }, 'GNX', 'Kendrick Lamar'), + ('discogs', { + 'id': 1, 'title': 'GNX', + 'artists': [{'name': 'Kendrick Lamar'}], + 'year': 2024, + }, 'GNX', 'Kendrick Lamar'), + ('musicbrainz', { + 'id': 'mbid', 'title': 'GNX', + 'artist-credit': [{'artist': {'name': 'Kendrick Lamar'}}], + }, 'GNX', 'Kendrick Lamar'), + ('hydrabase', { + 'id': 'hb', 'name': 'GNX', + 'artists': [{'name': 'Kendrick Lamar'}], + }, 'GNX', 'Kendrick Lamar'), + ('qobuz', { + 'id': 1, 'title': 'GNX', + 'artist': {'id': 2, 'name': 'Kendrick Lamar'}, + 'tracks_count': 12, + }, 'GNX', 'Kendrick Lamar'), +]) +def test_typed_path_works_for_every_registered_source(source, raw, expected_name, expected_artist): + """Each of the seven registered providers should round-trip through + the typed path producing usable output.""" + info = album_tracks._build_album_info( + raw, album_id='whatever', source=source, + ) + assert info['name'] == expected_name + assert info['artist'] == expected_artist + assert info['source'] == source diff --git a/tests/metadata/test_discography_typed_path.py b/tests/metadata/test_discography_typed_path.py new file mode 100644 index 00000000..9b664dca --- /dev/null +++ b/tests/metadata/test_discography_typed_path.py @@ -0,0 +1,160 @@ +"""Pin the typed-path migration in `core/metadata/discography.py`. + +`_build_discography_release_dict` and `_build_artist_detail_release_card` +historically did duck-typed extraction with fallback chains. This pr +routes them through `Album.from__dict()` when caller supplies +a known source. Legacy duck-typing kicks in as fallback. + +These tests pin: +- Typed path used when source is recognized. +- Typed path output matches expected fields the legacy path produced. +- Legacy path runs unchanged when source is empty/unknown OR when + the typed converter raises. +- Cross-provider parametrized smoke for every registered source. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from core.metadata import discography + + +SAMPLE_SPOTIFY_RELEASE = { + 'id': 'sp123', + 'name': 'GNX', + 'artists': [{'id': 'kdot', 'name': 'Kendrick Lamar'}], + 'release_date': '2024-11-22', + 'total_tracks': 12, + 'album_type': 'album', + 'images': [{'url': 'https://i.scdn.co/640.jpg', 'height': 640}], + 'external_urls': {'spotify': 'https://open.spotify.com/album/sp123'}, +} + + +# --------------------------------------------------------------------------- +# _build_discography_release_dict +# --------------------------------------------------------------------------- + + +def test_typed_path_used_for_known_source(): + out = discography._build_discography_release_dict( + SAMPLE_SPOTIFY_RELEASE, artist_id='kdot', source='spotify', + ) + assert out['id'] == 'sp123' + assert out['name'] == 'GNX' + assert out['artist_name'] == 'Kendrick Lamar' + assert out['release_date'] == '2024-11-22' + assert out['album_type'] == 'album' + assert out['total_tracks'] == 12 + assert out['image_url'] == 'https://i.scdn.co/640.jpg' + assert out['external_urls'] == {'spotify': 'https://open.spotify.com/album/sp123'} + + +def test_legacy_path_used_when_no_source(): + out = discography._build_discography_release_dict( + SAMPLE_SPOTIFY_RELEASE, artist_id='kdot', + ) + assert out['id'] == 'sp123' + assert out['name'] == 'GNX' + assert out['artist_name'] == 'Kendrick Lamar' + + +def test_legacy_path_used_for_unknown_source(): + out = discography._build_discography_release_dict( + SAMPLE_SPOTIFY_RELEASE, artist_id='kdot', source='made_up', + ) + assert out['id'] == 'sp123' + + +def test_legacy_path_used_when_typed_converter_raises(): + def _explode(_): + raise RuntimeError('simulated converter bug') + + with patch.dict(discography._TYPED_ALBUM_CONVERTERS, + {'spotify': _explode}): + out = discography._build_discography_release_dict( + SAMPLE_SPOTIFY_RELEASE, artist_id='kdot', source='spotify', + ) + # Legacy path still produced a result. + assert out['id'] == 'sp123' + assert out['name'] == 'GNX' + + +def test_release_with_no_id_returns_none(): + raw = dict(SAMPLE_SPOTIFY_RELEASE) + raw.pop('id') + out = discography._build_discography_release_dict( + raw, artist_id='kdot', source='spotify', + ) + assert out is None + + +@pytest.mark.parametrize('source,raw', [ + ('itunes', { + 'collectionId': 1, 'collectionName': 'GNX', + 'artistName': 'Kendrick Lamar', 'trackCount': 12, + }), + ('deezer', { + 'id': 1, 'title': 'GNX', + 'artist': {'name': 'Kendrick Lamar'}, 'nb_tracks': 12, + }), + ('discogs', { + 'id': 1, 'title': 'GNX', + 'artists': [{'name': 'Kendrick Lamar'}], 'year': 2024, + }), + ('musicbrainz', { + 'id': 'mbid', 'title': 'GNX', + 'artist-credit': [{'artist': {'name': 'Kendrick Lamar'}}], + }), + ('hydrabase', { + 'id': 'hb', 'name': 'GNX', + 'artists': [{'name': 'Kendrick Lamar'}], + }), + ('qobuz', { + 'id': 1, 'title': 'GNX', + 'artist': {'name': 'Kendrick Lamar'}, 'tracks_count': 12, + }), +]) +def test_typed_path_works_for_every_registered_source(source, raw): + out = discography._build_discography_release_dict( + raw, artist_id='whatever', source=source, + ) + assert out is not None + assert out['name'] == 'GNX' + + +# --------------------------------------------------------------------------- +# _build_artist_detail_release_card — typed dispatch on raw input +# --------------------------------------------------------------------------- + + +def test_artist_detail_card_typed_path(): + card = discography._build_artist_detail_release_card( + SAMPLE_SPOTIFY_RELEASE, source='spotify', + ) + assert card['id'] == 'sp123' + assert card['name'] == 'GNX' + assert card['album_type'] == 'album' + assert card['year'] == '2024' + assert card['release_date'] == '2024-11-22' + assert card['image_url'] == 'https://i.scdn.co/640.jpg' + assert card['track_count'] == 12 + + +def test_artist_detail_card_legacy_path_no_source(): + """Existing canonical-dict input (no source) takes legacy path.""" + canonical = { + 'id': 'sp123', + 'name': 'GNX', + 'album_type': 'album', + 'release_date': '2024-11-22', + 'image_url': 'https://i.scdn.co/640.jpg', + 'total_tracks': 12, + } + card = discography._build_artist_detail_release_card(canonical) + assert card['id'] == 'sp123' + assert card['name'] == 'GNX' + assert card['year'] == '2024' diff --git a/tests/metadata/test_enrichment_events.py b/tests/metadata/test_enrichment_events.py index 73d77942..b3afc7bb 100644 --- a/tests/metadata/test_enrichment_events.py +++ b/tests/metadata/test_enrichment_events.py @@ -22,11 +22,11 @@ WORKERS = [ # Endpoint URLs keyed by worker name ENDPOINTS = { - 'musicbrainz': '/api/musicbrainz/status', - 'audiodb': '/api/audiodb/status', - 'deezer': '/api/deezer/status', - 'spotify-enrichment': '/api/spotify-enrichment/status', - 'itunes-enrichment': '/api/itunes-enrichment/status', + 'musicbrainz': '/api/enrichment/musicbrainz/status', + 'audiodb': '/api/enrichment/audiodb/status', + 'deezer': '/api/enrichment/deezer/status', + 'spotify-enrichment': '/api/enrichment/spotify/status', + 'itunes-enrichment': '/api/enrichment/itunes/status', 'hydrabase': '/api/hydrabase-worker/status', 'repair': '/api/repair/status', } diff --git a/tests/metadata/test_image_url_normalization.py b/tests/metadata/test_image_url_normalization.py new file mode 100644 index 00000000..2832ba4c --- /dev/null +++ b/tests/metadata/test_image_url_normalization.py @@ -0,0 +1,44 @@ +"""Regression tests for image URL normalization.""" + +from __future__ import annotations + +import pytest +from urllib.parse import quote + + +@pytest.mark.parametrize( + "thumb_url", + [ + "/api/image-proxy?url=https%3A%2F%2Fexample.com%2Fcover.jpg", + "http://host.docker.internal:4533/api/image-proxy?u=ketiska&t=abc&s=def&v=1.16.1&c=SoulSync&f=json", + ], +) +def test_normalize_image_url_leaves_existing_image_proxy_urls_alone(thumb_url): + """Existing proxy URLs should not be wrapped in a second proxy layer.""" + from core.metadata import normalize_image_url + + assert normalize_image_url(thumb_url) == thumb_url + + +def test_normalize_image_url_proxies_internal_http_urls(monkeypatch): + """Raw internal image URLs should still be routed through SoulSync's proxy.""" + from core.metadata import normalize_image_url + from core.metadata import artwork + + class _FakeConfig: + def get_active_media_server(self): + return "spotify" + + def get_plex_config(self): + return {} + + def get_jellyfin_config(self): + return {} + + def get_navidrome_config(self): + return {} + + monkeypatch.setattr(artwork, "get_config_manager", lambda: _FakeConfig()) + + url = "http://localhost:4533/cover.jpg" + assert normalize_image_url(url) == f"/api/image-proxy?url={quote(url, safe='')}" diff --git a/tests/metadata/test_metadata_enrichment.py b/tests/metadata/test_metadata_enrichment.py index 83644fdf..3255e8c1 100644 --- a/tests/metadata/test_metadata_enrichment.py +++ b/tests/metadata/test_metadata_enrichment.py @@ -8,6 +8,25 @@ import requests from core.metadata import enrichment as me from core.metadata import artwork as ma from core.metadata import source as ms +from core.metadata import album_mbid_cache as _album_mbid_cache + + +@pytest.fixture(autouse=True) +def _isolate_persistent_album_mbid_cache(monkeypatch): + """The MB release lookup in `core/metadata/source.py` consults the + persistent album-MBID cache (`core/metadata/album_mbid_cache.py`) + before calling MusicBrainz. Tests in this file pin per-test MB + call counts and in-memory cache state — they shouldn't get + bypassed by leftover persistent rows from other tests sharing the + same SQLite database. + + Easiest fix: monkeypatch the persistent cache to be a no-op for + these tests. They focus on the in-memory layer + MB call shape; + the persistent layer has its own dedicated tests at + `tests/metadata/test_album_mbid_cache.py`. + """ + monkeypatch.setattr(_album_mbid_cache, 'lookup', lambda *a, **kw: None) + monkeypatch.setattr(_album_mbid_cache, 'record', lambda *a, **kw: False) class _Config: diff --git a/tests/metadata/test_metadata_gap_filler.py b/tests/metadata/test_metadata_gap_filler.py index 06a8a263..cd87bcdc 100644 --- a/tests/metadata/test_metadata_gap_filler.py +++ b/tests/metadata/test_metadata_gap_filler.py @@ -127,7 +127,7 @@ def _make_context(conn): report_progress=lambda *args, **kwargs: None, sleep_or_stop=lambda seconds: False, mb_client=_FakeMBClient(), - create_finding=lambda **kwargs: findings.append(kwargs), + create_finding=lambda **kwargs: (findings.append(kwargs) or True), findings=findings, ) diff --git a/tests/metadata/test_runtime_bundle.py b/tests/metadata/test_runtime_bundle.py index 415944af..8b6bed3e 100644 --- a/tests/metadata/test_runtime_bundle.py +++ b/tests/metadata/test_runtime_bundle.py @@ -23,6 +23,7 @@ def test_build_import_pipeline_runtime_exposes_expected_contract(): "deezer_worker", "audiodb_worker", "tidal_client", + "hifi_client", "qobuz_enrichment_worker", "lastfm_worker", "genius_worker", @@ -38,6 +39,7 @@ def test_build_metadata_enrichment_runtime_exposes_expected_contract(): "deezer_worker": object(), "audiodb_worker": object(), "tidal_client": object(), + "hifi_client": object(), "qobuz_enrichment_worker": object(), "lastfm_worker": object(), "genius_worker": object(), diff --git a/tests/metadata/test_typed_metadata_types.py b/tests/metadata/test_typed_metadata_types.py new file mode 100644 index 00000000..20478f75 --- /dev/null +++ b/tests/metadata/test_typed_metadata_types.py @@ -0,0 +1,572 @@ +"""Pin the per-provider Album converter contracts. + +Each provider returns its own response shape. The +``Album.from__dict()`` classmethods are the SINGLE place +that knows that shape. Consumers must be able to trust that an +``Album`` instance has the same field semantics regardless of which +provider it came from. + +These tests use realistic sample payloads (truncated from real API +responses) and pin: +- Required fields are always populated even when source data is + partial or messy (defaults applied uniformly). +- Cross-provider field semantics match — e.g. ``release_date`` is + always 'YYYY' or 'YYYY-MM-DD' regardless of whether Spotify gave + us 'YYYY-MM-DD', iTunes gave us '2024-01-15T00:00:00Z', or + Discogs gave us a bare year integer. +- Provider-specific quirks are normalized at the converter boundary + (Discogs `(N)` disambiguation suffix, iTunes `100x100bb` artwork + URLs, Deezer's nested `artist` object). +- ``to_context_dict()`` produces the canonical SoulSync-internal + shape consumers currently expect. + +When a future PR adds a new provider, this file is where the +contract test goes. +""" + +from __future__ import annotations + +import pytest + +from core.metadata.types import Album, Artist, Track + + +# --------------------------------------------------------------------------- +# Spotify +# --------------------------------------------------------------------------- + + +def test_album_from_spotify_dict_full_response(): + """A typical /albums/{id} response — populated fields, full track list.""" + raw = { + 'id': '0hvT3yIEysuuvkK73vgdcW', + 'name': 'GNX', + 'artists': [ + {'id': '2YZyLoL8N0Wb9xBt1NhZWg', 'name': 'Kendrick Lamar'}, + ], + 'release_date': '2024-11-22', + 'total_tracks': 12, + 'album_type': 'album', + 'images': [ + {'url': 'https://i.scdn.co/image/abc123', 'height': 640, 'width': 640}, + ], + 'genres': ['hip hop', 'rap'], + 'label': 'pgLang/Interscope', + 'external_ids': {'upc': '00602465123456'}, + 'external_urls': {'spotify': 'https://open.spotify.com/album/0hvT3yIEysuuvkK73vgdcW'}, + } + + album = Album.from_spotify_dict(raw) + + assert album.id == '0hvT3yIEysuuvkK73vgdcW' + assert album.name == 'GNX' + assert album.artists == ['Kendrick Lamar'] + assert album.artist_id == '2YZyLoL8N0Wb9xBt1NhZWg' + assert album.release_date == '2024-11-22' + assert album.total_tracks == 12 + assert album.album_type == 'album' + assert album.image_url == 'https://i.scdn.co/image/abc123' + assert album.genres == ['hip hop', 'rap'] + assert album.label == 'pgLang/Interscope' + assert album.barcode == '00602465123456' + assert album.source == 'spotify' + assert album.external_ids == {'spotify': '0hvT3yIEysuuvkK73vgdcW', 'upc': '00602465123456'} + + +def test_album_from_spotify_dict_handles_missing_fields(): + """Defensive: minimal payload still produces a valid Album.""" + raw = {'id': 'x', 'name': 'Y'} + album = Album.from_spotify_dict(raw) + assert album.id == 'x' + assert album.name == 'Y' + assert album.artists == ['Unknown Artist'] + assert album.release_date == '' + assert album.total_tracks == 0 + assert album.album_type == 'album' + assert album.image_url is None + assert album.label is None + + +def test_album_from_spotify_dict_multi_artist(): + """Featured artists / collabs — all names captured, primary artist + id is the first one.""" + raw = { + 'id': 'a1', + 'name': 'Luther', + 'artists': [ + {'id': 'kdot', 'name': 'Kendrick Lamar'}, + {'id': 'sza', 'name': 'SZA'}, + ], + 'total_tracks': 1, + } + album = Album.from_spotify_dict(raw) + assert album.artists == ['Kendrick Lamar', 'SZA'] + assert album.artist_id == 'kdot' + + +# --------------------------------------------------------------------------- +# iTunes +# --------------------------------------------------------------------------- + + +def test_album_from_itunes_dict_full_response(): + raw = { + 'collectionId': 1782145638, + 'collectionName': 'GNX', + 'artistName': 'Kendrick Lamar', + 'artistId': 368183298, + 'releaseDate': '2024-11-22T08:00:00Z', + 'trackCount': 12, + 'collectionType': 'Album', + 'artworkUrl100': 'https://is1.mzstatic.com/image/100x100bb.jpg', + 'collectionViewUrl': 'https://music.apple.com/album/gnx/1782145638', + 'primaryGenreName': 'Hip-Hop/Rap', + } + album = Album.from_itunes_dict(raw) + assert album.id == '1782145638' + assert album.name == 'GNX' + assert album.artists == ['Kendrick Lamar'] + # iTunes ISO timestamp truncated to date + assert album.release_date == '2024-11-22' + assert album.total_tracks == 12 + assert album.album_type == 'album' + # 100x100bb upgraded to 3000x3000bb + assert album.image_url == 'https://is1.mzstatic.com/image/3000x3000bb.jpg' + assert album.artist_id == '368183298' + assert album.genres == ['Hip-Hop/Rap'] + assert album.source == 'itunes' + assert album.external_ids['itunes'] == '1782145638' + assert album.external_ids['itunes_artist'] == '368183298' + + +def test_album_from_itunes_dict_infers_album_type_from_track_count(): + """iTunes doesn't tag album type — convert per the existing + heuristic (1-3 single, 4-6 EP, 7+ album).""" + base = {'collectionId': 1, 'collectionName': 'X', 'artistName': 'A', + 'collectionType': 'Album'} + assert Album.from_itunes_dict({**base, 'trackCount': 1}).album_type == 'single' + assert Album.from_itunes_dict({**base, 'trackCount': 5}).album_type == 'ep' + assert Album.from_itunes_dict({**base, 'trackCount': 12}).album_type == 'album' + + +def test_album_from_itunes_dict_detects_compilation(): + raw = {'collectionId': 1, 'collectionName': 'Best Of', 'artistName': 'V/A', + 'collectionType': 'Compilation', 'trackCount': 20} + assert Album.from_itunes_dict(raw).album_type == 'compilation' + + +def test_album_from_itunes_dict_strips_single_ep_suffix(): + """iTunes appends ' - Single' / ' - EP' to single/EP collection + names. Strip so cross-provider matching works on the actual title.""" + raw = {'collectionId': 1, 'collectionName': 'Track Name - Single', + 'artistName': 'A', 'trackCount': 1} + assert Album.from_itunes_dict(raw).name == 'Track Name' + + +# --------------------------------------------------------------------------- +# Deezer +# --------------------------------------------------------------------------- + + +def test_album_from_deezer_dict_full_response(): + raw = { + 'id': 12345, + 'title': 'GNX', + 'artist': {'id': 67890, 'name': 'Kendrick Lamar'}, + 'release_date': '2024-11-22', + 'nb_tracks': 12, + 'record_type': 'album', + 'cover_xl': 'https://e-cdns-images.dzcdn.net/images/cover/abc/1000x1000-000000-80-0-0.jpg', + 'genres': {'data': [{'id': 116, 'name': 'Rap/Hip Hop'}]}, + 'label': 'pgLang', + 'upc': '00602465123456', + 'link': 'https://www.deezer.com/album/12345', + } + album = Album.from_deezer_dict(raw) + assert album.id == '12345' + assert album.name == 'GNX' + assert album.artists == ['Kendrick Lamar'] + assert album.artist_id == '67890' + assert album.release_date == '2024-11-22' + assert album.total_tracks == 12 + assert album.album_type == 'album' + assert 'cover/abc' in album.image_url + assert album.genres == ['Rap/Hip Hop'] + assert album.label == 'pgLang' + assert album.barcode == '00602465123456' + assert album.source == 'deezer' + + +def test_album_from_deezer_dict_falls_back_through_cover_sizes(): + """Deezer cover URLs come in xl/big/medium/small variants. Prefer xl.""" + base = {'id': 1, 'title': 'X', 'artist': {'name': 'A'}} + # xl present + a = Album.from_deezer_dict({**base, 'cover_xl': 'XL', 'cover_big': 'BIG'}) + assert a.image_url == 'XL' + # only big + b = Album.from_deezer_dict({**base, 'cover_big': 'BIG'}) + assert b.image_url == 'BIG' + # nothing + c = Album.from_deezer_dict(base) + assert c.image_url is None + + +# --------------------------------------------------------------------------- +# Discogs +# --------------------------------------------------------------------------- + + +def test_album_from_discogs_dict_full_response(): + raw = { + 'id': 33445566, + 'title': 'GNX', + 'artists': [ + {'id': 1234, 'name': 'Kendrick Lamar'}, + ], + 'year': 2024, + 'tracklist': [ + {'position': 'A1', 'title': 'wacced out murals', 'type_': 'track'}, + {'position': 'A2', 'title': 'squabble up', 'type_': 'track'}, + {'position': 'B1', 'title': 'luther', 'type_': 'track'}, + ], + 'images': [ + {'type': 'primary', 'uri': 'https://img.discogs.com/abc.jpg', 'uri150': 'https://img.discogs.com/abc-150.jpg'}, + ], + 'genres': ['Hip Hop'], + 'styles': ['Conscious'], + 'labels': [{'name': 'pgLang', 'catno': 'PG001'}], + 'identifiers': [ + {'type': 'Barcode', 'value': '00602465123456'}, + {'type': 'Other', 'value': 'XYZ'}, + ], + 'uri': 'https://www.discogs.com/release/33445566', + } + album = Album.from_discogs_dict(raw) + assert album.id == '33445566' + assert album.name == 'GNX' + assert album.artists == ['Kendrick Lamar'] + assert album.artist_id == '1234' + assert album.release_date == '2024' + assert album.total_tracks == 3 + assert album.album_type == 'album' + # uri preferred over uri150 + assert album.image_url == 'https://img.discogs.com/abc.jpg' + # Discogs genres + styles merged + assert 'Hip Hop' in album.genres and 'Conscious' in album.genres + assert album.label == 'pgLang' + assert album.barcode == '00602465123456' + assert album.source == 'discogs' + + +def test_album_from_discogs_dict_strips_disambiguation_suffix(): + """`Madonna (3)` → `Madonna` so cross-provider matches work.""" + raw = {'id': 1, 'title': 'Y', 'artists': [{'name': 'Madonna (3)'}]} + album = Album.from_discogs_dict(raw) + assert album.artists == ['Madonna'] + + +def test_album_from_discogs_dict_year_zero_means_unknown(): + """Discogs `year=0` is the sentinel for unknown — empty release_date.""" + raw = {'id': 1, 'title': 'Y', 'artists': [{'name': 'X'}], 'year': 0} + assert Album.from_discogs_dict(raw).release_date == '' + + +def test_album_from_discogs_dict_counts_only_track_type_entries(): + """Discogs tracklists include heading rows, indices, etc (type_='heading'). + Only count actual tracks (type_='track').""" + raw = { + 'id': 1, 'title': 'Y', 'artists': [{'name': 'X'}], + 'tracklist': [ + {'title': 'Side A', 'type_': 'heading'}, + {'title': 'Track 1', 'type_': 'track'}, + {'title': 'Track 2', 'type_': 'track'}, + {'title': 'Side B', 'type_': 'heading'}, + {'title': 'Track 3', 'type_': 'track'}, + ], + } + assert Album.from_discogs_dict(raw).total_tracks == 3 + + +# --------------------------------------------------------------------------- +# MusicBrainz +# --------------------------------------------------------------------------- + + +def test_album_from_musicbrainz_dict_full_response(): + raw = { + 'id': 'abc-123-mbid', + 'title': 'GNX', + 'artist-credit': [ + {'artist': {'id': 'kdot-mbid', 'name': 'Kendrick Lamar'}}, + ], + 'date': '2024-11-22', + 'media': [{'track-count': 12}], + 'release-group': { + 'id': 'rg-mbid', + 'primary-type': 'Album', + }, + 'label-info': [{'label': {'name': 'pgLang'}}], + 'barcode': '00602465123456', + } + album = Album.from_musicbrainz_dict(raw) + assert album.id == 'abc-123-mbid' + assert album.name == 'GNX' + assert album.artists == ['Kendrick Lamar'] + assert album.artist_id == 'kdot-mbid' + assert album.release_date == '2024-11-22' + assert album.total_tracks == 12 + assert album.album_type == 'album' + assert album.label == 'pgLang' + assert album.barcode == '00602465123456' + assert album.external_ids['musicbrainz'] == 'abc-123-mbid' + assert album.external_ids['musicbrainz_release_group'] == 'rg-mbid' + + +def test_album_from_musicbrainz_dict_sums_multi_disc_tracks(): + """MB stores per-disc track counts; total = sum across media.""" + raw = { + 'id': 'x', 'title': 'Multi Disc', + 'artist-credit': [{'artist': {'name': 'A'}}], + 'media': [{'track-count': 14}, {'track-count': 5}], + } + assert Album.from_musicbrainz_dict(raw).total_tracks == 19 + + +def test_album_from_musicbrainz_dict_release_group_type_overrides_default(): + raw = { + 'id': 'x', 'title': 'X', + 'artist-credit': [{'artist': {'name': 'A'}}], + 'release-group': {'id': 'rg', 'primary-type': 'Single'}, + 'media': [{'track-count': 1}], + } + assert Album.from_musicbrainz_dict(raw).album_type == 'single' + + +# --------------------------------------------------------------------------- +# Qobuz +# --------------------------------------------------------------------------- + + +def test_album_from_qobuz_dict_full_response(): + raw = { + 'id': 12345, + 'title': 'GNX', + 'artist': {'id': 67890, 'name': 'Kendrick Lamar'}, + 'release_date_original': '2024-11-22', + 'released_at': '2024-11-22T08:00:00', + 'tracks_count': 12, + 'image': { + 'small': 'https://qobuz/small.jpg', + 'large': 'https://qobuz/large.jpg', + 'thumbnail': 'https://qobuz/thumb.jpg', + }, + 'genre': {'id': 116, 'name': 'Hip-Hop/Rap'}, + 'label': {'id': 999, 'name': 'pgLang'}, + 'upc': '00602465123456', + 'url': 'https://www.qobuz.com/album/gnx/12345', + } + album = Album.from_qobuz_dict(raw) + assert album.id == '12345' + assert album.name == 'GNX' + assert album.artists == ['Kendrick Lamar'] + assert album.artist_id == '67890' + assert album.release_date == '2024-11-22' + assert album.total_tracks == 12 + assert album.image_url == 'https://qobuz/large.jpg' + assert album.genres == ['Hip-Hop/Rap'] + assert album.label == 'pgLang' + assert album.barcode == '00602465123456' + assert album.source == 'qobuz' + + +def test_album_from_qobuz_dict_falls_back_through_image_sizes(): + base = {'id': 1, 'title': 'X', 'artist': {'name': 'A'}} + a = Album.from_qobuz_dict({**base, 'image': {'small': 'S'}}) + assert a.image_url == 'S' + b = Album.from_qobuz_dict({**base, 'image': {}}) + assert b.image_url is None + + +def test_album_from_qobuz_dict_strips_iso_timestamp_to_date(): + raw = {'id': 1, 'title': 'X', 'artist': {'name': 'A'}, + 'released_at': '2024-11-22T08:00:00'} + assert Album.from_qobuz_dict(raw).release_date == '2024-11-22' + + +# --------------------------------------------------------------------------- +# Tidal +# --------------------------------------------------------------------------- + + +def test_album_from_tidal_object_full_shape(): + """tidalapi returns objects, not dicts. Use SimpleNamespace stand-ins + to mirror the tidalapi.Album shape.""" + from types import SimpleNamespace + + artist_obj = SimpleNamespace(id=67890, name='Kendrick Lamar') + album_obj = SimpleNamespace( + id=12345, + name='GNX', + artist=artist_obj, + release_date='2024-11-22', + num_tracks=12, + type='ALBUM', + picture='abc-123-def', + universal_product_number='00602465123456', + image=lambda size=640: f'https://resources.tidal.com/images/abc/123/def/{size}x{size}.jpg', + ) + + album = Album.from_tidal_object(album_obj) + assert album.id == '12345' + assert album.name == 'GNX' + assert album.artists == ['Kendrick Lamar'] + assert album.artist_id == '67890' + assert album.release_date == '2024-11-22' + assert album.total_tracks == 12 + assert album.album_type == 'album' # lowercased + assert album.image_url and 'tidal.com' in album.image_url + assert album.barcode == '00602465123456' + assert album.source == 'tidal' + assert album.external_ids['tidal'] == '12345' + + +def test_album_from_tidal_object_falls_back_to_picture_url_when_image_method_missing(): + from types import SimpleNamespace + album_obj = SimpleNamespace( + id=1, name='X', + artist=SimpleNamespace(name='A', id=2), + release_date='2024', + num_tracks=10, + picture='aa-bb-cc', + ) + album = Album.from_tidal_object(album_obj) + assert album.image_url and 'aa/bb/cc' in album.image_url + + +def test_album_from_tidal_object_handles_missing_attrs(): + """Bare-minimum tidalapi-shaped object — should still produce a + valid Album with sensible defaults.""" + from types import SimpleNamespace + album_obj = SimpleNamespace(id=1, name='X', artist=None) + album = Album.from_tidal_object(album_obj) + assert album.id == '1' + assert album.name == 'X' + assert album.artists == ['Unknown Artist'] + assert album.total_tracks == 0 + assert album.album_type == 'album' + assert album.image_url is None + + +# --------------------------------------------------------------------------- +# Hydrabase +# --------------------------------------------------------------------------- + + +def test_album_from_hydrabase_dict_full_response(): + raw = { + 'id': 'soul-12345', + 'name': 'GNX', + 'artists': [{'id': 'soul-artist-1', 'name': 'Kendrick Lamar'}], + 'release_date': '2024-11-22', + 'total_tracks': 12, + 'album_type': 'album', + 'image_url': 'https://hydrabase.example/cover.jpg', + 'soul_id': 'soul-12345', + 'artist_id': 'soul-artist-1', + } + album = Album.from_hydrabase_dict(raw) + assert album.id == 'soul-12345' + assert album.name == 'GNX' + assert album.artists == ['Kendrick Lamar'] + assert album.artist_id == 'soul-artist-1' + assert album.image_url == 'https://hydrabase.example/cover.jpg' + assert album.source == 'hydrabase' + assert album.external_ids['hydrabase'] == 'soul-12345' + assert album.external_ids['soul'] == 'soul-12345' + + +def test_album_from_hydrabase_dict_handles_string_artists(): + """Hydrabase responses sometimes return artists as a flat list of + name strings, sometimes as dicts. Both shapes work.""" + raw_str = {'id': '1', 'name': 'X', 'artists': ['Artist A']} + assert Album.from_hydrabase_dict(raw_str).artists == ['Artist A'] + + raw_dict = {'id': '1', 'name': 'X', 'artists': [{'name': 'Artist B'}]} + assert Album.from_hydrabase_dict(raw_dict).artists == ['Artist B'] + + +# --------------------------------------------------------------------------- +# Cross-provider invariants +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('factory,raw', [ + ('from_spotify_dict', {'id': 'x', 'name': 'X'}), + ('from_itunes_dict', {'collectionId': 1, 'collectionName': 'X', 'artistName': 'A'}), + ('from_deezer_dict', {'id': 1, 'title': 'X', 'artist': {'name': 'A'}}), + ('from_discogs_dict', {'id': 1, 'title': 'X', 'artists': [{'name': 'A'}]}), + ('from_musicbrainz_dict', {'id': 'x', 'title': 'X', + 'artist-credit': [{'artist': {'name': 'A'}}]}), + ('from_hydrabase_dict', {'id': 'x', 'name': 'X', 'artists': [{'name': 'A'}]}), + ('from_qobuz_dict', {'id': 1, 'title': 'X', 'artist': {'name': 'A'}}), +]) +def test_every_converter_produces_required_fields(factory, raw): + """Every converter MUST populate the required fields with sensible + defaults even on minimal input. This is the contract consumers + rely on to drop their fallback chains.""" + album = getattr(Album, factory)(raw) + assert isinstance(album.id, str) and album.id + assert isinstance(album.name, str) and album.name + assert isinstance(album.artists, list) and len(album.artists) >= 1 + assert isinstance(album.release_date, str) # may be empty + assert isinstance(album.total_tracks, int) + assert isinstance(album.album_type, str) and album.album_type + assert isinstance(album.genres, list) + assert isinstance(album.external_ids, dict) + assert isinstance(album.external_urls, dict) + assert album.source # always set by converter + + +@pytest.mark.parametrize('factory,raw', [ + ('from_spotify_dict', {'id': 'x', 'name': 'X'}), + ('from_itunes_dict', {'collectionId': 1, 'collectionName': 'X', 'artistName': 'A'}), + ('from_deezer_dict', {'id': 1, 'title': 'X', 'artist': {'name': 'A'}}), + ('from_discogs_dict', {'id': 1, 'title': 'X', 'artists': [{'name': 'A'}]}), + ('from_musicbrainz_dict', {'id': 'x', 'title': 'X', + 'artist-credit': [{'artist': {'name': 'A'}}]}), + ('from_hydrabase_dict', {'id': 'x', 'name': 'X', 'artists': [{'name': 'A'}]}), + ('from_qobuz_dict', {'id': 1, 'title': 'X', 'artist': {'name': 'A'}}), +]) +def test_to_context_dict_shape_is_uniform_across_providers(factory, raw): + """The bridge dict every consumer currently expects has the same + shape regardless of provider. Pin so a future converter change + can't subtly break consumer expectations.""" + album = getattr(Album, factory)(raw) + ctx = album.to_context_dict() + + expected_keys = { + 'id', 'name', 'artist', 'artist_name', 'artist_id', 'artists', + 'image_url', 'images', 'release_date', 'album_type', + 'total_tracks', 'source', 'genres', 'label', 'barcode', + 'external_ids', 'external_urls', + } + assert set(ctx.keys()) == expected_keys + + +# --------------------------------------------------------------------------- +# Track / Artist — light coverage; full converters land in a follow-up PR +# --------------------------------------------------------------------------- + + +def test_track_dataclass_required_fields(): + t = Track(id='1', name='X', artists=['A'], album='Y', duration_ms=1000) + assert t.id == '1' + assert t.popularity == 0 # default + assert t.external_ids == {} + + +def test_artist_dataclass_required_fields(): + a = Artist(id='1', name='X') + assert a.id == '1' + assert a.followers == 0 # default + assert a.genres == [] diff --git a/tests/search/test_search_orchestrator.py b/tests/search/test_search_orchestrator.py index 25082404..02766746 100644 --- a/tests/search/test_search_orchestrator.py +++ b/tests/search/test_search_orchestrator.py @@ -129,7 +129,7 @@ def _build_deps(**overrides): spotify_client=None, hydrabase_client=None, hydrabase_worker=None, - soulseek_client=None, + download_orchestrator=None, fix_artist_image_url=lambda u: f'FIXED::{u}' if u else None, is_hydrabase_active=lambda: False, get_metadata_fallback_source=lambda: 'spotify', @@ -462,24 +462,27 @@ class _FakeYouTube: class _FakeSoulseekWithYT: def __init__(self, youtube): - self.youtube = youtube + self._youtube = youtube + + def client(self, name): + return self._youtube if name == 'youtube' else None def test_resolve_youtube_videos_returns_subclient(): yt = _FakeYouTube() - deps = _build_deps(soulseek_client=_FakeSoulseekWithYT(yt)) + deps = _build_deps(download_orchestrator=_FakeSoulseekWithYT(yt)) assert orchestrator.resolve_youtube_videos_client(deps) is yt def test_resolve_youtube_videos_no_soulseek_returns_none(): - deps = _build_deps(soulseek_client=None) + deps = _build_deps(download_orchestrator=None) assert orchestrator.resolve_youtube_videos_client(deps) is None def test_resolve_youtube_videos_no_youtube_attr_returns_none(): class _NoYT: pass - deps = _build_deps(soulseek_client=_NoYT()) + deps = _build_deps(download_orchestrator=_NoYT()) assert orchestrator.resolve_youtube_videos_client(deps) is None diff --git a/tests/search/test_search_stream.py b/tests/search/test_search_stream.py index fc12f724..6404d343 100644 --- a/tests/search/test_search_stream.py +++ b/tests/search/test_search_stream.py @@ -53,15 +53,20 @@ class _FakeStreamClient: class _FakeSoulseek: def __init__(self, youtube=None, tidal=None, qobuz=None, hifi=None, deezer_dl=None, lidarr=None, results_per_query=None): - self.youtube = youtube - self.tidal = tidal - self.qobuz = qobuz - self.hifi = hifi - self.deezer_dl = deezer_dl - self.lidarr = lidarr + self._clients = { + 'youtube': youtube, + 'tidal': tidal, + 'qobuz': qobuz, + 'hifi': hifi, + 'deezer_dl': deezer_dl, + 'lidarr': lidarr, + } self._results = results_per_query or {} self.search_calls = [] + def client(self, name): + return self._clients.get(name) + async def search(self, query, timeout=15): self.search_calls.append(query) return self._results.get(query, ([], [])) @@ -164,7 +169,7 @@ def test_stream_finds_match_on_first_query(): result = stream.stream_search_track( track_name='Money', artist_name='Pink Floyd', album_name=None, duration_ms=180000, - config_manager=cfg, soulseek_client=soul, matching_engine=engine, + config_manager=cfg, download_orchestrator=soul, matching_engine=engine, run_async=_run_async, ) assert result is not None @@ -185,7 +190,7 @@ def test_stream_walks_to_second_query_on_no_match(): result = stream.stream_search_track( track_name='Money (Remastered)', artist_name='Pink Floyd', album_name=None, duration_ms=180000, - config_manager=cfg, soulseek_client=soul, matching_engine=engine, + config_manager=cfg, download_orchestrator=soul, matching_engine=engine, run_async=_run_async, ) assert result is not None @@ -204,7 +209,7 @@ def test_stream_returns_none_when_no_matches(): result = stream.stream_search_track( track_name='Money', artist_name='Pink Floyd', album_name=None, duration_ms=180000, - config_manager=cfg, soulseek_client=soul, matching_engine=engine, + config_manager=cfg, download_orchestrator=soul, matching_engine=engine, run_async=_run_async, ) assert result is None @@ -212,7 +217,7 @@ def test_stream_returns_none_when_no_matches(): def test_stream_falls_back_to_default_soulseek_when_no_direct_client(): # Effective mode = 'youtube' (coerced from soulseek), but soul.youtube is None - # → code falls through to soulseek_client.search directly. Streaming-mode + # → code falls through to download_orchestrator.search directly. Streaming-mode # query gen → "Pink Floyd Money". soul = _FakeSoulseek(results_per_query={'Pink Floyd Money': ([object()], [])}) cfg = _FakeConfig({ @@ -224,7 +229,7 @@ def test_stream_falls_back_to_default_soulseek_when_no_direct_client(): result = stream.stream_search_track( track_name='Money', artist_name='Pink Floyd', album_name=None, duration_ms=180000, - config_manager=cfg, soulseek_client=soul, matching_engine=engine, + config_manager=cfg, download_orchestrator=soul, matching_engine=engine, run_async=_run_async, ) assert result is not None @@ -252,7 +257,7 @@ def test_stream_continues_past_per_query_exception(): result = stream.stream_search_track( track_name='Money (Live)', artist_name='Pink Floyd', album_name=None, duration_ms=180000, - config_manager=cfg, soulseek_client=soul, matching_engine=engine, + config_manager=cfg, download_orchestrator=soul, matching_engine=engine, run_async=_run_async, ) # First query raised, second succeeded diff --git a/tests/streaming/test_prepare.py b/tests/streaming/test_prepare.py index cfe3c277..bbfc9cef 100644 --- a/tests/streaming/test_prepare.py +++ b/tests/streaming/test_prepare.py @@ -10,7 +10,7 @@ from core.streaming import prepare as sp class _FakeSoulseek: - """Minimal soulseek_client stub for the stream-prep worker.""" + """Minimal download_orchestrator stub for the stream-prep worker.""" def __init__(self, *, download_id='dl-1', all_downloads=None): self._download_id = download_id @@ -43,7 +43,7 @@ def _build_deps( state = state if state is not None else {} deps = sp.PrepareStreamDeps( config_manager=type('C', (), {'get': lambda self, k, d=None: d})(), - soulseek_client=soulseek or _FakeSoulseek(), + download_orchestrator=soulseek or _FakeSoulseek(), stream_lock=threading.Lock(), project_root=project_root, docker_resolve_path=lambda p: p, @@ -107,7 +107,7 @@ def test_stream_folder_cleared_before_download(tmp_path): # --------------------------------------------------------------------------- def test_download_returns_none_marks_error(tmp_path): - """soulseek_client.download() returning None → state.error.""" + """download_orchestrator.download() returning None → state.error.""" sk = _FakeSoulseek(download_id=None) deps = _build_deps(soulseek=sk, project_root=str(tmp_path)) diff --git a/tests/test_acoustid_skip_logic.py b/tests/test_acoustid_skip_logic.py new file mode 100644 index 00000000..272ca7d7 --- /dev/null +++ b/tests/test_acoustid_skip_logic.py @@ -0,0 +1,204 @@ +"""Tighten the AcoustID "language/script" skip exemption. + +User report (Mr. Morale download): three different track requests +(Rich Interlude, Savior Interlude, Savior) each received the same +WRONG audio file (Kendrick's R.O.T.C Interlude from his 2010 mixtape). +AcoustID flagged the title mismatch but the verification logic +SKIPPED rather than FAILED with the reason "likely same song in +different language/script." + +The old condition was: + best_score >= 0.95 AND (title_sim >= 0.55 OR artist_sim >= match) + +That OR-clause fired for English-vs-English titles by the same artist +that share NO actual content — same artist + word "interlude" in both +titles cleared the bar. The skip then trusted the wrong file as +correct. + +New condition: only skip when there's positive evidence the mismatch +is a transliteration / language-script case: +- (a) Either side of the comparison contains non-ASCII characters AND + artist matches strongly. Real cases: Japanese kanji ↔ romaji, + Korean hangul ↔ romaji, etc. +- (b) BOTH title AND artist similarity are very high (>=0.80, ARTIST + threshold). Real cases: title differs only by punctuation / + casing that fell below strict-match thresholds. + +For English-vs-English with very different titles by the same artist, +the skip no longer fires — verification correctly returns FAIL, +quarantining the wrong file. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from core.acoustid_verification import ( + AcoustIDVerification, + VerificationResult, +) + + +@pytest.fixture +def verifier(monkeypatch): + """A verifier with the network/fingerprint side stubbed so we can + drive the title/artist comparison logic directly.""" + v = AcoustIDVerification() + + # Stub availability check to avoid touching real chromaprint + class _StubClient: + def is_available(self): + return True, 'available' + + def fingerprint_and_lookup(self, path): + # Each test injects its own desired return value via + # monkeypatch on this method; default is empty. + return None + + v.acoustid_client = _StubClient() + return v + + +def _stub_lookup(verifier, *, recordings, best_score): + """Make `fingerprint_and_lookup` return a fabricated AcoustID result.""" + verifier.acoustid_client.fingerprint_and_lookup = lambda path: { + 'recordings': recordings, + 'best_score': best_score, + 'recording_mbids': [r.get('id') for r in recordings if r.get('id')], + } + + +# --------------------------------------------------------------------------- +# The headline regression — Rich Interlude vs R.O.T.C Interlude +# --------------------------------------------------------------------------- + + +def test_english_titles_same_artist_no_longer_skipped(verifier): + """User's actual case: requested 'Rich (Interlude)' by Kendrick + Lamar, AcoustID identified the file as 'R.O.T.C. (interlude)' by + Kendrick Lamar. Same artist, same word 'interlude', but completely + different songs. Old skip-logic let it pass; new logic must FAIL + so the file gets quarantined.""" + _stub_lookup(verifier, recordings=[ + {'title': 'R.O.T.C. (interlude)', 'artist': 'Kendrick Lamar feat. BJ the Chicago Kid'}, + ], best_score=0.96) + + result, msg = verifier.verify_audio_file( + '/fake/path.flac', + 'Rich (Interlude)', + 'Kendrick Lamar', + ) + assert result == VerificationResult.FAIL + # Message should be the wrong-file message, NOT the language/script skip + assert 'mismatch' in msg.lower() + assert 'language/script' not in msg.lower() + + +def test_savior_request_returning_rotc_no_longer_skipped(verifier): + """Same bug surface, different track. Confirms the fix isn't + Rich-Interlude-specific.""" + _stub_lookup(verifier, recordings=[ + {'title': 'R.O.T.C. (interlude)', 'artist': 'Kendrick Lamar feat. BJ the Chicago Kid'}, + ], best_score=0.96) + + result, _msg = verifier.verify_audio_file( + '/fake/path.flac', + 'Savior', + 'Kendrick Lamar', + ) + assert result == VerificationResult.FAIL + + +# --------------------------------------------------------------------------- +# The legitimate skip cases — must STILL fire +# --------------------------------------------------------------------------- + + +def test_japanese_kanji_to_romaji_still_skipped(verifier): + """Real language/script case: AcoustID's database has the kanji + title, the user requested the romaji version. Same artist (in + Latin script), high fingerprint confidence. Skip should still + fire so a correct file isn't false-quarantined.""" + _stub_lookup(verifier, recordings=[ + {'title': '残酷な天使のテーゼ', 'artist': 'Yoko Takahashi'}, + ], best_score=0.97) + + result, msg = verifier.verify_audio_file( + '/fake/path.flac', + 'Zankoku na Tenshi no Theze', + 'Yoko Takahashi', + ) + assert result == VerificationResult.SKIP + assert 'language/script' in msg.lower() + + +def test_minor_punctuation_difference_passes_outright(verifier): + """Punctuation-only difference: both 'MAAD' and 'M.A.A.D' normalize + similarly enough that the strict TITLE_MATCH_THRESHOLD is met and + verification PASSES (better outcome than SKIP). Pin this so a + future tightening of the strict thresholds doesn't accidentally + push these into the FAIL bucket.""" + _stub_lookup(verifier, recordings=[ + {'title': 'M.A.A.D City', 'artist': 'Kendrick Lamar'}, + ], best_score=0.97) + + result, _msg = verifier.verify_audio_file( + '/fake/path.flac', + 'MAAD City', + 'Kendrick Lamar', + ) + # PASS or SKIP both fine — the critical assertion is "not FAIL". + assert result != VerificationResult.FAIL + + +def test_low_fingerprint_score_never_skipped(verifier): + """Below the 0.95 confidence floor, the skip exemption should + never fire — even for plausibly-real language/script cases. We + don't have enough signal to be sure the audio matches.""" + _stub_lookup(verifier, recordings=[ + {'title': '残酷な天使のテーゼ', 'artist': 'Yoko Takahashi'}, + ], best_score=0.80) # below 0.95 floor + + result, _msg = verifier.verify_audio_file( + '/fake/path.flac', + 'Zankoku na Tenshi no Theze', + 'Yoko Takahashi', + ) + assert result == VerificationResult.FAIL + + +def test_high_score_but_artist_mismatch_no_longer_skipped(verifier): + """Even with high fingerprint AND non-ASCII chars present, if the + artist DOESN'T match well, we don't have enough signal to skip. + Could be a cover by a different artist.""" + _stub_lookup(verifier, recordings=[ + {'title': '残酷な天使のテーゼ', 'artist': 'Some Other Singer'}, + ], best_score=0.97) + + result, _msg = verifier.verify_audio_file( + '/fake/path.flac', + 'Zankoku na Tenshi no Theze', + 'Yoko Takahashi', + ) + assert result == VerificationResult.FAIL + + +def test_old_loose_threshold_no_longer_fires_for_unrelated_titles(verifier): + """Pin the negative case for the old loose threshold (title_sim + >= 0.55). 'Crown' vs 'Crown of Thorns' had similarity around 0.6 + in some normalizations — under old logic with high confidence + and matching artist that would skip. New logic requires title_sim + >= 0.80 OR non-ASCII presence.""" + _stub_lookup(verifier, recordings=[ + {'title': 'Crown of Thorns', 'artist': 'Kendrick Lamar'}, + ], best_score=0.96) + + result, _msg = verifier.verify_audio_file( + '/fake/path.flac', + 'Crown', + 'Kendrick Lamar', + ) + # User asked for 'Crown', got 'Crown of Thorns' — should FAIL now + assert result == VerificationResult.FAIL diff --git a/tests/test_acoustid_version_mismatch.py b/tests/test_acoustid_version_mismatch.py new file mode 100644 index 00000000..9d542763 --- /dev/null +++ b/tests/test_acoustid_version_mismatch.py @@ -0,0 +1,186 @@ +"""AcoustID verification rejects version mismatches (instrumental / live / etc). + +Discord report (corruption [BWC]): downloads coming through as instrumentals +when the user expected vocal versions. Slipped past AcoustID verification +because the ``_normalize`` step strips parentheticals and version-suffix +tags ("(Instrumental)", "- Live", etc) so that legitimate name variations +don't fail the title-similarity check. Side effect: "Song Name" and +"Song Name (Instrumental)" both normalize to "song name", title sim is +1.0, file passes verification despite being the wrong cut. + +Fix: detect the version label on each side BEFORE normalization runs. +If the expected and matched versions disagree, return FAIL — the +fingerprint did identify a real song, just not the version the caller +asked for. + +Reuses ``MusicMatchingEngine.detect_version_type`` so the same patterns +that the pre-download Soulseek matcher uses also drive post-download +verification (no duplicated regex tables). +""" + +from __future__ import annotations + +import pytest + +from core.acoustid_verification import ( + AcoustIDVerification, + VerificationResult, +) + + +@pytest.fixture +def verifier(): + """Verifier with the network/fingerprint side stubbed so tests can + drive the title/artist comparison logic directly.""" + v = AcoustIDVerification() + + class _StubClient: + def is_available(self): + return True, "available" + + def fingerprint_and_lookup(self, path): + return None # tests inject via _stub_lookup below + + v.acoustid_client = _StubClient() + return v + + +def _stub_lookup(verifier, *, recordings, best_score=0.95): + verifier.acoustid_client.fingerprint_and_lookup = lambda path: { + "recordings": recordings, + "best_score": best_score, + "recording_mbids": [r.get("id") for r in recordings if r.get("id")], + } + + +# --------------------------------------------------------------------------- +# The headline bug — instrumental returned where vocal was expected. +# --------------------------------------------------------------------------- + + +def test_instrumental_returned_for_vocal_request_fails(verifier): + """User asked for a vocal track; file's fingerprint matched an + instrumental version of the same song. Old normalizer stripped + "(Instrumental)" and let it pass. Must FAIL now.""" + _stub_lookup(verifier, recordings=[ + {"title": "In My Feelings (Instrumental)", "artist": "Drake"}, + ]) + + result, msg = verifier.verify_audio_file( + "/fake/path.flac", + "In My Feelings", + "Drake", + ) + assert result == VerificationResult.FAIL + assert "version mismatch" in msg.lower() + assert "instrumental" in msg.lower() + + +def test_instrumental_request_with_vocal_file_fails(verifier): + """Symmetric case: user asked for the instrumental cut explicitly, + file's fingerprint matched the regular vocal version. Also FAIL — + they're different recordings.""" + _stub_lookup(verifier, recordings=[ + {"title": "In My Feelings", "artist": "Drake"}, + ]) + + result, _ = verifier.verify_audio_file( + "/fake/path.flac", + "In My Feelings (Instrumental)", + "Drake", + ) + assert result == VerificationResult.FAIL + + +# --------------------------------------------------------------------------- +# Different-version mismatches (live vs acoustic, etc) — also FAIL. +# --------------------------------------------------------------------------- + + +def test_different_versions_disagree_fails(verifier): + """Caller asked for the live cut; file is the acoustic cut. Both + are non-original versions, but they're different non-original + versions — must FAIL.""" + _stub_lookup(verifier, recordings=[ + {"title": "Hello (Acoustic)", "artist": "Adele"}, + ]) + + result, _ = verifier.verify_audio_file( + "/fake/path.flac", + "Hello (Live at Wembley)", + "Adele", + ) + assert result == VerificationResult.FAIL + + +# --------------------------------------------------------------------------- +# Regression guards — version match must NOT cause false-FAIL. +# --------------------------------------------------------------------------- + + +def test_original_to_original_passes(verifier): + """Plain track to plain track — no version on either side. Verify + the version gate doesn't get in the way of the normal happy path.""" + _stub_lookup(verifier, recordings=[ + {"title": "Bohemian Rhapsody", "artist": "Queen"}, + ]) + + result, _ = verifier.verify_audio_file( + "/fake/path.flac", + "Bohemian Rhapsody", + "Queen", + ) + assert result == VerificationResult.PASS + + +def test_matching_versions_pass(verifier): + """Both expected and matched are the live version of the same song. + Versions agree — must PASS.""" + _stub_lookup(verifier, recordings=[ + {"title": "Hello (Live at Wembley)", "artist": "Adele"}, + ]) + + result, _ = verifier.verify_audio_file( + "/fake/path.flac", + "Hello (Live at Wembley)", + "Adele", + ) + assert result == VerificationResult.PASS + + +# --------------------------------------------------------------------------- +# Secondary scan path — version gate must apply to fallback recordings too. +# --------------------------------------------------------------------------- + + +def test_secondary_scan_skips_wrong_version_recordings(verifier): + """When the best AcoustID recording's title doesn't match strongly + enough, verify scans through the rest of the recordings list looking + for a better candidate. That fallback path must also reject + wrong-version variants — otherwise an instrumental from the same + fingerprint cluster could win the scan and pass verification.""" + _stub_lookup(verifier, recordings=[ + # Best by combined score: a different track entirely. Same + # artist, completely different title. Original version (no + # version mismatch on this one). + {"title": "Some Other Song", "artist": "Drake"}, + # Fallback candidate: instrumental version of the requested + # song. Without the scan-loop version gate, this would PASS + # (title matches after stripping "(Instrumental)", artist + # matches). With the gate, it gets skipped and the loop falls + # through. + {"title": "In My Feelings (Instrumental)", "artist": "Drake"}, + ]) + + result, _ = verifier.verify_audio_file( + "/fake/path.flac", + "In My Feelings", + "Drake", + ) + # Best-recording version check passes (both 'original'), then the + # main pass/fail bucket misses (title doesn't match), fallback scan + # skips the instrumental, no other valid recording → falls through + # to the final unmatched logic. Either FAIL or SKIP is acceptable + # here; the critical assertion is "did NOT pass via the + # instrumental-version recording". + assert result != VerificationResult.PASS diff --git a/tests/test_album_completeness_job.py b/tests/test_album_completeness_job.py index 05f5736b..8d58a03c 100644 --- a/tests/test_album_completeness_job.py +++ b/tests/test_album_completeness_job.py @@ -528,7 +528,7 @@ def test_scan_uses_cached_api_track_count_without_expected_total_lookup(monkeypa job._get_expected_total = spy findings = [] - context = _make_job_context(db, create_finding=lambda **kwargs: findings.append(kwargs)) + context = _make_job_context(db, create_finding=lambda **kwargs: (findings.append(kwargs) or True)) result = job.scan(context) @@ -568,7 +568,7 @@ def test_scan_falls_back_to_api_and_persists_count_on_cache_miss(monkeypatch): ) findings = [] - context = _make_job_context(db, create_finding=lambda **kwargs: findings.append(kwargs)) + context = _make_job_context(db, create_finding=lambda **kwargs: (findings.append(kwargs) or True)) job = AlbumCompletenessJob() result = job.scan(context) @@ -606,7 +606,7 @@ def test_scan_ignores_track_count_completely(monkeypatch): ) findings = [] - context = _make_job_context(db, create_finding=lambda **kwargs: findings.append(kwargs)) + context = _make_job_context(db, create_finding=lambda **kwargs: (findings.append(kwargs) or True)) job = AlbumCompletenessJob() result = job.scan(context) diff --git a/tests/test_album_mbid_consistency.py b/tests/test_album_mbid_consistency.py new file mode 100644 index 00000000..396c1c90 --- /dev/null +++ b/tests/test_album_mbid_consistency.py @@ -0,0 +1,355 @@ +"""Tests for the album MBID consistency detector + fix action. + +User report (Samuel [KC]): tracks of the same album sometimes carry +different ``MUSICBRAINZ_ALBUMID`` tags, which causes Navidrome to split +the album into multiple entries. The detector groups tracks by DB album, +finds the consensus (most-common) album MBID, and flags dissenting +tracks. The fix action rewrites the dissenter's tag to match. + +Tests cover: +- The Picard-standard tag read/write helpers across MP3 / FLAC / OGG +- Detector behavior: agreement → no flags, single dissenter → flag, + ties → no flag (no clear consensus to fix toward), tracks without + album MBID skipped, single-track albums skipped, no album_id skipped. +- Fix action: rewrites the tag, surfaces error on missing file / + missing consensus. + +Real audio files (FLAC + MP3 + OGG) are generated with mutagen so we +exercise the actual tag write/read path, not just helper logic. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from core.repair_jobs import mbid_mismatch_detector as detector +from core.repair_jobs.mbid_mismatch_detector import ( + MbidMismatchDetectorJob, + _read_album_mbid_from_file, + _write_album_mbid_to_file, +) + + +# --------------------------------------------------------------------------- +# Audio file fabrication +# --------------------------------------------------------------------------- + + +def _make_minimal_flac(path: Path) -> None: + """Create a real FLAC file with mutagen so we can read/write tags.""" + from mutagen.flac import FLAC, StreamInfo + # Write minimal FLAC bytes — mutagen needs a real file to attach tags. + # Use a tiny synthesized FLAC: stream marker + STREAMINFO block + 1 + # frame's worth of silence. Simpler: write a base FLAC the official + # way. + import struct + fLaC = b'fLaC' + # Minimum STREAMINFO: 16 bits min/max block size, 24 bits min/max + # frame size, 20 bits sample rate, 3 bits channels-1, 5 bits + # bits-per-sample-1, 36 bits total samples, 128 bits md5 sig. + streaminfo = bytearray(34) + # Write enough so mutagen accepts it + streaminfo[0:2] = struct.pack('>H', 4096) # min block + streaminfo[2:4] = struct.pack('>H', 4096) # max block + streaminfo[10] = 0x0A # sample rate / channels (won't validate strictly) + streaminfo[12] = 0x70 # bits-per-sample bits + # Block header: last_block=1, type=0 (STREAMINFO), length=34 + block_header = bytes([0x80, 0x00, 0x00, 0x22]) + path.write_bytes(fLaC + block_header + bytes(streaminfo)) + # Verify mutagen can open it + audio = FLAC(str(path)) + audio.save() + + +def _make_minimal_mp3(path: Path) -> None: + """Create a real MP3 file with an empty ID3 tag block.""" + from mutagen.id3 import ID3 + # MP3 frame header: 0xFF FB 90 64 + silence frame. + # Easier approach: write empty bytes + ID3 init. + # Use a longer plausible MP3 frame so mutagen doesn't choke. + mp3_frame = b'\xff\xfb\x90\x64' + (b'\x00' * 417) + path.write_bytes(mp3_frame * 4) + # Initialize an empty ID3 tag block so add() works later + try: + tags = ID3() + tags.save(str(path)) + except Exception: + # Some mutagen versions require an existing audio file + from mutagen.mp3 import MP3 + audio = MP3(str(path)) + audio.add_tags() + audio.save() + + +def _make_minimal_ogg(path: Path) -> None: + """Create a real Ogg Vorbis file. mutagen ships a tiny stub helper.""" + # Easiest path: write the stripped-down Ogg Vorbis header bytes. + # In practice this is fragile, so we just generate a 1-second silent + # vorbis using the bare minimum mutagen accepts. Skip if unavailable. + pytest.skip("Ogg synthesis requires libvorbis; covered via FLAC + MP3") + + +# --------------------------------------------------------------------------- +# Tag read/write helpers +# --------------------------------------------------------------------------- + + +def test_read_album_mbid_returns_none_for_missing_tag(tmp_path: Path) -> None: + f = tmp_path / 'no_tag.flac' + _make_minimal_flac(f) + assert _read_album_mbid_from_file(str(f)) is None + + +def test_write_then_read_album_mbid_flac(tmp_path: Path) -> None: + """Round-trip the album MBID through a real FLAC file using the + Picard-standard MUSICBRAINZ_ALBUMID Vorbis comment.""" + f = tmp_path / 'track.flac' + _make_minimal_flac(f) + target_mbid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' + + assert _write_album_mbid_to_file(str(f), target_mbid) is True + assert _read_album_mbid_from_file(str(f)) == target_mbid + + +def test_write_album_mbid_overwrites_existing_flac(tmp_path: Path) -> None: + """Writing the same tag twice should leave only the latest value + (no duplicate Vorbis entries).""" + f = tmp_path / 'track.flac' + _make_minimal_flac(f) + _write_album_mbid_to_file(str(f), 'old-mbid') + _write_album_mbid_to_file(str(f), 'new-mbid') + + from mutagen.flac import FLAC + audio = FLAC(str(f)) + vals = audio.get('MUSICBRAINZ_ALBUMID', []) + assert vals == ['new-mbid'] + + +# Note: MP3 round-trip tests skipped — synthesizing a valid MPEG audio +# frame in pure Python is fragile (mutagen can't sync to fake frames). +# The MP3 ID3 path uses mutagen's standard add()/get() on TXXX frames, +# which is exhaustively tested in mutagen itself. The Picard-standard +# tag descriptor (`MusicBrainz Album Id`) is asserted via the +# _ALBUM_MBID_TAG_KEYS module constant test below, and the write/clear +# logic is structurally identical to the FLAC path covered above. + + +def test_album_mbid_tag_keys_match_picard_standards() -> None: + """The constants written into files must match exactly what Picard + writes — mismatch causes media servers to read no MBID at all.""" + from core.repair_jobs.mbid_mismatch_detector import _ALBUM_MBID_TAG_KEYS + assert _ALBUM_MBID_TAG_KEYS['mp3_txxx_desc'] == 'MusicBrainz Album Id' + assert _ALBUM_MBID_TAG_KEYS['vorbis'] == 'MUSICBRAINZ_ALBUMID' + assert _ALBUM_MBID_TAG_KEYS['mp4'] == '----:com.apple.iTunes:MusicBrainz Album Id' + + +def test_read_album_mbid_returns_none_for_unreadable_file(tmp_path: Path) -> None: + """Defensive: garbage file shouldn't raise.""" + f = tmp_path / 'broken.flac' + f.write_bytes(b'not actually flac') + assert _read_album_mbid_from_file(str(f)) is None + + +def test_write_album_mbid_returns_false_for_unreadable_file(tmp_path: Path) -> None: + f = tmp_path / 'broken.flac' + f.write_bytes(b'not actually flac') + assert _write_album_mbid_to_file(str(f), 'mbid') is False + + +def test_write_album_mbid_returns_false_for_empty_input(tmp_path: Path) -> None: + f = tmp_path / 'track.flac' + _make_minimal_flac(f) + assert _write_album_mbid_to_file(str(f), '') is False + + +# --------------------------------------------------------------------------- +# Detector — _scan_album_mbid_consistency +# --------------------------------------------------------------------------- + + +def _build_tracks_in_db(tmp_path: Path, *, + album_id: int, + track_specs: list, + artist_name: str = 'Kendrick Lamar', + album_title: str = 'GNX') -> list: + """Produce a list of fake DB rows + create the FLAC files on disk + with the requested MUSICBRAINZ_ALBUMID values. + + `track_specs` is a list of (track_id, embedded_album_mbid_or_None). + Pass None for tracks that should have no embedded MBID at all. + """ + rows = [] + for track_id, embedded_mbid in track_specs: + f = tmp_path / f'track_{track_id}.flac' + _make_minimal_flac(f) + if embedded_mbid is not None: + _write_album_mbid_to_file(str(f), embedded_mbid) + rows.append({ + 'id': track_id, + 'title': f'Track {track_id}', + 'album_id': album_id, + 'file_path': str(f), + 'artist_name': artist_name, + 'album_title': album_title, + 'album_thumb': None, + 'artist_thumb': None, + }) + return rows + + +def _build_context(rows: list, tmp_path: Path) -> SimpleNamespace: + """Build a JobContext-shaped object that returns `rows` from the + DB query. Tracks calls to `create_finding` so tests can assert.""" + findings_created = [] + + class _FakeRow(dict): + def __getitem__(self, key): + return super().__getitem__(key) + + fake_rows = [_FakeRow(r) for r in rows] + + class _FakeCursor: + def execute(self, *a, **kw): + pass + def fetchall(self): + return fake_rows + + class _FakeConn: + def cursor(self): + return _FakeCursor() + def close(self): + pass + + class _FakeDB: + def _get_connection(self): + return _FakeConn() + + def _check_stop(): + return False + + def _create_finding(**kwargs): + findings_created.append(kwargs) + # Mirror real `_create_finding` contract: True on insert. + return True + + ctx = SimpleNamespace( + db=_FakeDB(), + transfer_folder=str(tmp_path), + config_manager=None, + check_stop=_check_stop, + report_progress=None, + create_finding=_create_finding, + findings=findings_created, + ) + return ctx + + +def _build_result() -> SimpleNamespace: + return SimpleNamespace(findings_created=0, errors=0) + + +def test_consistency_scan_creates_no_findings_when_all_match(tmp_path: Path) -> None: + rows = _build_tracks_in_db(tmp_path, album_id=10, track_specs=[ + (1, 'mbid-A'), (2, 'mbid-A'), (3, 'mbid-A'), + ]) + ctx = _build_context(rows, tmp_path) + result = _build_result() + job = MbidMismatchDetectorJob() + + with patch.object(detector, '_resolve_file_path', side_effect=lambda p, *a, **kw: p): + job._scan_album_mbid_consistency(ctx, result, download_folder='') + + assert ctx.findings == [] + assert result.findings_created == 0 + + +def test_consistency_scan_flags_lone_dissenter(tmp_path: Path) -> None: + """11 tracks agree on mbid-A, 1 track has mbid-B → flag the 1.""" + rows = _build_tracks_in_db(tmp_path, album_id=10, track_specs=[ + (i, 'mbid-A') for i in range(1, 12) + ] + [(99, 'mbid-B')]) + ctx = _build_context(rows, tmp_path) + result = _build_result() + job = MbidMismatchDetectorJob() + + with patch.object(detector, '_resolve_file_path', side_effect=lambda p, *a, **kw: p): + job._scan_album_mbid_consistency(ctx, result, download_folder='') + + assert len(ctx.findings) == 1 + f = ctx.findings[0] + assert f['finding_type'] == 'album_mbid_mismatch' + assert f['entity_id'] == '99' + assert f['details']['wrong_mbid'] == 'mbid-B' + assert f['details']['consensus_mbid'] == 'mbid-A' + assert f['details']['consensus_count'] == 11 + + +def test_consistency_scan_skips_single_track_albums(tmp_path: Path) -> None: + """Single-track album can't have a consistency issue.""" + rows = _build_tracks_in_db(tmp_path, album_id=10, track_specs=[(1, 'mbid-A')]) + ctx = _build_context(rows, tmp_path) + result = _build_result() + job = MbidMismatchDetectorJob() + + with patch.object(detector, '_resolve_file_path', side_effect=lambda p, *a, **kw: p): + job._scan_album_mbid_consistency(ctx, result, download_folder='') + + assert ctx.findings == [] + + +def test_consistency_scan_skips_tracks_without_album_mbid(tmp_path: Path) -> None: + """Tracks with NO embedded album MBID don't break Navidrome — they + just don't participate in the consistency check. Don't flag them + and don't let them count toward consensus.""" + rows = _build_tracks_in_db(tmp_path, album_id=10, track_specs=[ + (1, 'mbid-A'), + (2, 'mbid-A'), + (3, None), # no MBID — should be ignored + ]) + ctx = _build_context(rows, tmp_path) + result = _build_result() + job = MbidMismatchDetectorJob() + + with patch.object(detector, '_resolve_file_path', side_effect=lambda p, *a, **kw: p): + job._scan_album_mbid_consistency(ctx, result, download_folder='') + + assert ctx.findings == [] + + +def test_consistency_scan_skips_when_no_clear_consensus(tmp_path: Path) -> None: + """If 2 tracks have mbid-A and 2 have mbid-B (tied), there's no + clear consensus to fix toward. Flag nothing — surface as a manual + decision.""" + rows = _build_tracks_in_db(tmp_path, album_id=10, track_specs=[ + (1, 'mbid-A'), (2, 'mbid-A'), + (3, 'mbid-B'), (4, 'mbid-B'), + ]) + ctx = _build_context(rows, tmp_path) + result = _build_result() + job = MbidMismatchDetectorJob() + + with patch.object(detector, '_resolve_file_path', side_effect=lambda p, *a, **kw: p): + job._scan_album_mbid_consistency(ctx, result, download_folder='') + + assert ctx.findings == [] + + +def test_consistency_scan_handles_unresolvable_file_path(tmp_path: Path) -> None: + """If a track's file_path can't be resolved (resolver returns None), + skip silently — don't crash.""" + rows = _build_tracks_in_db(tmp_path, album_id=10, track_specs=[ + (1, 'mbid-A'), (2, 'mbid-A'), (3, 'mbid-B'), + ]) + ctx = _build_context(rows, tmp_path) + result = _build_result() + job = MbidMismatchDetectorJob() + + # Unresolvable for everything + with patch.object(detector, '_resolve_file_path', return_value=None): + job._scan_album_mbid_consistency(ctx, result, download_folder='') + + assert ctx.findings == [] diff --git a/tests/test_artist_top_tracks_clients.py b/tests/test_artist_top_tracks_clients.py new file mode 100644 index 00000000..fcb2e33f --- /dev/null +++ b/tests/test_artist_top_tracks_clients.py @@ -0,0 +1,218 @@ +"""Tests for the new artist top-tracks client methods. + +Issue #513: surface an artist's "top X popular songs" for one-click download +without pulling the entire discography. Spotify and Deezer expose this via +native APIs; iTunes / Discogs / MusicBrainz don't, so the frontend falls +back to the existing Last.fm display-only sidebar. + +Scope: client methods only. The Flask endpoint that wraps them is small +enough (source dispatch + DB id resolution + JSON response) that +exercising the underlying client methods is the load-bearing test layer. +A full-app Flask test client wasn't worth pulling in here — importing +``web_server`` at test-collection time spins up worker threads that race +with caplog-using tests elsewhere in the suite. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from core.deezer_client import DeezerClient +from core.spotify_client import SpotifyClient + + +# --------------------------------------------------------------------------- +# Spotify client method +# --------------------------------------------------------------------------- + + +def test_spotify_get_artist_top_tracks_returns_track_list(monkeypatch): + """Wraps spotipy's `artist_top_tracks` and returns the `tracks` array.""" + client = SpotifyClient.__new__(SpotifyClient) + fake_sp = MagicMock() + fake_sp.artist_top_tracks.return_value = { + 'tracks': [ + {'id': 't1', 'name': 'Song A', 'artists': [{'name': 'Artist'}]}, + {'id': 't2', 'name': 'Song B', 'artists': [{'name': 'Artist'}]}, + {'id': 't3', 'name': 'Song C', 'artists': [{'name': 'Artist'}]}, + ] + } + client.sp = fake_sp + monkeypatch.setattr(client, 'is_spotify_authenticated', lambda: True) + + result = client.get_artist_top_tracks('artist_id', country='US', limit=10) + + assert len(result) == 3 + assert [t['id'] for t in result] == ['t1', 't2', 't3'] + fake_sp.artist_top_tracks.assert_called_once_with('artist_id', country='US') + + +def test_spotify_get_artist_top_tracks_honors_ui_limit(monkeypatch): + """Spotify always returns up to 10 tracks; the limit param is a UI trim only.""" + client = SpotifyClient.__new__(SpotifyClient) + fake_sp = MagicMock() + fake_sp.artist_top_tracks.return_value = { + 'tracks': [{'id': f't{i}', 'name': f'Song {i}'} for i in range(10)] + } + client.sp = fake_sp + monkeypatch.setattr(client, 'is_spotify_authenticated', lambda: True) + + result = client.get_artist_top_tracks('artist_id', limit=3) + assert len(result) == 3 + + +def test_spotify_get_artist_top_tracks_returns_empty_when_unauthed(monkeypatch): + """No API call should fire when Spotify isn't authenticated. Lets the + endpoint return `success=False, reason=spotify_not_authenticated` + instead of throwing.""" + client = SpotifyClient.__new__(SpotifyClient) + fake_sp = MagicMock() + client.sp = fake_sp + monkeypatch.setattr(client, 'is_spotify_authenticated', lambda: False) + + result = client.get_artist_top_tracks('artist_id') + assert result == [] + fake_sp.artist_top_tracks.assert_not_called() + + +def test_spotify_get_artist_top_tracks_returns_empty_when_artist_id_missing(monkeypatch): + """Defensive guard — no API call for empty/None artist id.""" + client = SpotifyClient.__new__(SpotifyClient) + fake_sp = MagicMock() + client.sp = fake_sp + monkeypatch.setattr(client, 'is_spotify_authenticated', lambda: True) + + assert client.get_artist_top_tracks('') == [] + assert client.get_artist_top_tracks(None) == [] + fake_sp.artist_top_tracks.assert_not_called() + + +def test_spotify_get_artist_top_tracks_swallows_api_errors(monkeypatch): + """Network/auth exceptions surface as empty list, not a crash — + the endpoint relies on this to fall through gracefully.""" + client = SpotifyClient.__new__(SpotifyClient) + fake_sp = MagicMock() + fake_sp.artist_top_tracks.side_effect = RuntimeError("boom") + client.sp = fake_sp + monkeypatch.setattr(client, 'is_spotify_authenticated', lambda: True) + + result = client.get_artist_top_tracks('artist_id') + assert result == [] + + +# --------------------------------------------------------------------------- +# Deezer client method +# --------------------------------------------------------------------------- + + +def test_deezer_get_artist_top_tracks_returns_spotify_compatible_shape(monkeypatch): + """Deezer's raw shape gets converted to the same dict layout the + Spotify endpoint produces (id, name, artists, album, duration_ms, + track_number, etc) so downstream code doesn't need to branch.""" + client = DeezerClient() + + raw_response = { + 'data': [ + { + 'id': 1001, + 'title': 'Some Hit', + 'duration': 200, # seconds + 'rank': 850000, + 'preview': 'https://example/preview.mp3', + 'link': 'https://deezer.com/track/1001', + 'track_position': 3, + 'disk_number': 1, + 'explicit_lyrics': False, + 'artist': {'id': 50, 'name': 'Test Artist'}, + 'album': { + 'id': 200, 'title': 'Greatest Hits', + 'cover_xl': 'https://example/cover_xl.jpg', + 'cover_big': 'https://example/cover_big.jpg', + }, + }, + ] + } + monkeypatch.setattr(client, '_api_get', lambda path, params=None: raw_response) + + tracks = client.get_artist_top_tracks('50', limit=5) + + assert len(tracks) == 1 + t = tracks[0] + assert t['id'] == '1001' + assert t['name'] == 'Some Hit' + assert t['duration_ms'] == 200_000 # converted to ms + assert t['track_number'] == 3 + assert t['disc_number'] == 1 + assert t['artists'] == [{'id': '50', 'name': 'Test Artist'}] + assert t['album']['id'] == '200' + assert t['album']['name'] == 'Greatest Hits' + assert t['album']['album_type'] == 'album' + assert any(img['url'] == 'https://example/cover_xl.jpg' for img in t['album']['images']) + assert t['_source'] == 'deezer' + + +def test_deezer_get_artist_top_tracks_empty_when_no_data(monkeypatch): + """Missing artist or empty response → empty list. Endpoint relies on + this to report `success=False, reason=no_tracks_found`.""" + client = DeezerClient() + monkeypatch.setattr(client, '_api_get', lambda path, params=None: None) + + assert client.get_artist_top_tracks('50') == [] + + +def test_deezer_get_artist_top_tracks_empty_when_artist_id_missing(monkeypatch): + """Defensive guard — no API call for empty artist id.""" + client = DeezerClient() + called = {'count': 0} + + def fake_api(*args, **kwargs): + called['count'] += 1 + return {'data': []} + + monkeypatch.setattr(client, '_api_get', fake_api) + + assert client.get_artist_top_tracks('') == [] + assert called['count'] == 0 + + +def test_deezer_get_artist_top_tracks_clamps_limit(monkeypatch): + """Limit param gets clamped at the upper bound (Deezer's max ~100) + and falls back to the default when the caller passes 0/None.""" + client = DeezerClient() + captured = {} + + def fake_api(path, params=None): + captured['params'] = params + return {'data': []} + + monkeypatch.setattr(client, '_api_get', fake_api) + + # Excessive limit clamped down to 100 + client.get_artist_top_tracks('50', limit=10000) + assert captured['params']['limit'] == 100 + + # 0 → falsy, falls back to default 10 (better than 1 — caller probably + # wanted "give me a sensible top-N", not "give me a single track") + client.get_artist_top_tracks('50', limit=0) + assert captured['params']['limit'] == 10 + + # Small valid limit passes through + client.get_artist_top_tracks('50', limit=3) + assert captured['params']['limit'] == 3 + + +def test_deezer_get_artist_top_tracks_skips_malformed_entries(monkeypatch): + """Defensive — non-dict entries in the response array get filtered out + rather than crashing the loop.""" + client = DeezerClient() + monkeypatch.setattr(client, '_api_get', lambda path, params=None: { + 'data': [ + None, # malformed: skipped + {'id': 1, 'title': 'Real', 'artist': {'name': 'A'}, 'album': {}}, + 'not a dict', # malformed: skipped + ] + }) + + tracks = client.get_artist_top_tracks('50') + assert len(tracks) == 1 + assert tracks[0]['name'] == 'Real' diff --git a/tests/test_create_finding_dedup_counter.py b/tests/test_create_finding_dedup_counter.py new file mode 100644 index 00000000..1327606e --- /dev/null +++ b/tests/test_create_finding_dedup_counter.py @@ -0,0 +1,301 @@ +"""Pin the bug fix where `_create_finding` silently dedup-skipped while +the caller's `findings_created` counter incremented anyway, causing +the maintenance job badge to inflate (e.g. "364 findings" displayed +when 0 new pending rows existed in the DB). + +Now `_create_finding` returns: +- True — a NEW pending row was inserted. +- False — dedup-skipped (an equivalent row already exists with status + pending/resolved/dismissed) OR a DB error occurred. + +Callers must only increment `findings_created` on True. Skipped-dedup +counter exposed separately as `findings_skipped_dedup` for log +transparency. +""" + +from __future__ import annotations + +import os +import sqlite3 +import tempfile +from unittest.mock import MagicMock + +import pytest + +from core.repair_jobs.base import JobResult +from core.repair_worker import RepairWorker + + +@pytest.fixture +def repair_worker_with_temp_db(): + """A RepairWorker wired to a temporary SQLite db with the + `repair_findings` table created. Returns the worker; tears the db + down after the test.""" + fd, path = tempfile.mkstemp(suffix='.db') + os.close(fd) + + conn = sqlite3.connect(path) + conn.execute(""" + CREATE TABLE repair_findings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL, + finding_type TEXT NOT NULL, + severity TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + entity_type TEXT, + entity_id TEXT, + file_path TEXT, + title TEXT, + description TEXT, + details_json TEXT, + user_action TEXT, + resolved_at TEXT, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.commit() + conn.close() + + db_mock = MagicMock() + db_mock._get_connection = lambda: sqlite3.connect(path) + + worker = RepairWorker.__new__(RepairWorker) + worker.db = db_mock + + yield worker, path + + try: + os.remove(path) + except OSError: + pass + + +def test_create_finding_returns_true_on_first_insert(repair_worker_with_temp_db): + worker, _ = repair_worker_with_temp_db + result = worker._create_finding( + job_id='dup_detector', + finding_type='duplicate_tracks', + severity='info', + entity_type='track', + entity_id='123', + file_path='/foo/bar.mp3', + title='Test finding', + description='desc', + ) + assert result is True + + +def test_create_finding_returns_false_on_dedup_pending(repair_worker_with_temp_db): + worker, _ = repair_worker_with_temp_db + # First insert + worker._create_finding( + job_id='dup_detector', finding_type='duplicate_tracks', + severity='info', entity_type='track', entity_id='123', + file_path='/foo/bar.mp3', title='T', description='D', + ) + # Re-call with same args — should dedup + result = worker._create_finding( + job_id='dup_detector', finding_type='duplicate_tracks', + severity='info', entity_type='track', entity_id='123', + file_path='/foo/bar.mp3', title='T', description='D', + ) + assert result is False + + +def test_create_finding_returns_false_on_dedup_dismissed(repair_worker_with_temp_db): + """The user dismissed a finding in a prior session. A new scan + that re-discovers the same issue must NOT increment the badge — + the row exists with status='dismissed'.""" + worker, path = repair_worker_with_temp_db + + # Seed the DB with a dismissed finding + conn = sqlite3.connect(path) + conn.execute(""" + INSERT INTO repair_findings + (job_id, finding_type, severity, status, entity_type, entity_id, + file_path, title, description) + VALUES (?, ?, 'info', 'dismissed', 'track', '123', '/foo/bar.mp3', 'T', 'D') + """, ('dup_detector', 'duplicate_tracks')) + conn.commit() + conn.close() + + result = worker._create_finding( + job_id='dup_detector', finding_type='duplicate_tracks', + severity='info', entity_type='track', entity_id='123', + file_path='/foo/bar.mp3', title='T', description='D', + ) + assert result is False + + +def test_create_finding_returns_false_on_dedup_resolved(repair_worker_with_temp_db): + """An auto-fix or manual repair previously resolved a finding. + Re-discovery of the same issue should NOT inflate the badge.""" + worker, path = repair_worker_with_temp_db + + conn = sqlite3.connect(path) + conn.execute(""" + INSERT INTO repair_findings + (job_id, finding_type, severity, status, entity_type, entity_id, + file_path, title, description) + VALUES (?, ?, 'info', 'resolved', 'track', '123', '/foo/bar.mp3', 'T', 'D') + """, ('dup_detector', 'duplicate_tracks')) + conn.commit() + conn.close() + + result = worker._create_finding( + job_id='dup_detector', finding_type='duplicate_tracks', + severity='info', entity_type='track', entity_id='123', + file_path='/foo/bar.mp3', title='T', description='D', + ) + assert result is False + + +def test_create_finding_inserts_again_when_distinct_entity(repair_worker_with_temp_db): + """A different entity (different track id) is a NEW finding — + must NOT be dedup-blocked by an unrelated finding's existence.""" + worker, _ = repair_worker_with_temp_db + worker._create_finding( + job_id='dup_detector', finding_type='duplicate_tracks', + severity='info', entity_type='track', entity_id='123', + file_path='/foo/bar.mp3', title='T', description='D', + ) + result = worker._create_finding( + job_id='dup_detector', finding_type='duplicate_tracks', + severity='info', entity_type='track', entity_id='456', + file_path='/foo/baz.mp3', title='T', description='D', + ) + assert result is True + + +# --------------------------------------------------------------------------- +# JobResult — findings_skipped_dedup field is exposed +# --------------------------------------------------------------------------- + + +def test_job_result_has_skipped_dedup_field(): + result = JobResult() + assert result.findings_skipped_dedup == 0 + result.findings_skipped_dedup += 1 + assert result.findings_skipped_dedup == 1 + + +# --------------------------------------------------------------------------- +# End-to-end pattern: caller counts only true inserts +# --------------------------------------------------------------------------- + + +def test_caller_pattern_counts_only_real_inserts(repair_worker_with_temp_db): + """Simulate a job loop calling create_finding 5 times for the + same finding. Only the FIRST should count toward findings_created; + the remaining 4 should count toward findings_skipped_dedup. Badge + must reflect 1, not 5.""" + worker, _ = repair_worker_with_temp_db + result = JobResult() + + for _ in range(5): + inserted = worker._create_finding( + job_id='dup_detector', finding_type='duplicate_tracks', + severity='info', entity_type='track', entity_id='123', + file_path='/foo/bar.mp3', title='T', description='D', + ) + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 + + assert result.findings_created == 1 + assert result.findings_skipped_dedup == 4 + + +# --------------------------------------------------------------------------- +# _get_pending_count_by_job — feeds the job-card "X pending" badge +# --------------------------------------------------------------------------- + + +def test_get_pending_count_by_job_returns_per_job_dict(repair_worker_with_temp_db): + """Pin: helper returns ``{job_id: pending_count}`` based on rows + with status='pending' only. Job-card badge uses this so it shows + CURRENT pending state instead of the historical + ``last_run.findings_created`` (which inflates after a bulk-fix + moves all findings to status='resolved').""" + worker, path = repair_worker_with_temp_db + + conn = sqlite3.connect(path) + cur = conn.cursor() + # 3 pending duplicate_tracks for dup_detector + cur.executemany(""" + INSERT INTO repair_findings (job_id, finding_type, severity, status, + entity_type, entity_id, file_path, title, description) + VALUES (?, 'duplicate_tracks', 'info', 'pending', 'track', ?, ?, 'T', 'D') + """, [ + ('dup_detector', '1', '/a'), + ('dup_detector', '2', '/b'), + ('dup_detector', '3', '/c'), + ]) + # 2 pending missing_cover_art for cover_art_filler + cur.executemany(""" + INSERT INTO repair_findings (job_id, finding_type, severity, status, + entity_type, entity_id, file_path, title, description) + VALUES ('cover_art_filler', 'missing_cover_art', 'info', 'pending', + 'album', ?, ?, 'T', 'D') + """, [('10', '/x'), ('11', '/y')]) + # 1 RESOLVED finding — must NOT be counted + cur.execute(""" + INSERT INTO repair_findings (job_id, finding_type, severity, status, + entity_type, entity_id, file_path, title, description) + VALUES ('dup_detector', 'duplicate_tracks', 'info', 'resolved', + 'track', '99', '/z', 'T', 'D') + """) + # 1 DISMISSED finding — must NOT be counted + cur.execute(""" + INSERT INTO repair_findings (job_id, finding_type, severity, status, + entity_type, entity_id, file_path, title, description) + VALUES ('cover_art_filler', 'missing_cover_art', 'info', 'dismissed', + 'album', '88', '/w', 'T', 'D') + """) + conn.commit() + conn.close() + + result = worker._get_pending_count_by_job() + + assert result == { + 'dup_detector': 3, + 'cover_art_filler': 2, + } + + +def test_get_pending_count_by_job_returns_empty_when_no_pending(repair_worker_with_temp_db): + """Pin: jobs with zero pending findings are absent from the dict + (caller handles missing key as 0).""" + worker, path = repair_worker_with_temp_db + + conn = sqlite3.connect(path) + cur = conn.cursor() + cur.execute(""" + INSERT INTO repair_findings (job_id, finding_type, severity, status, + entity_type, entity_id, file_path, title, description) + VALUES ('dup_detector', 'duplicate_tracks', 'info', 'resolved', + 'track', '1', '/a', 'T', 'D') + """) + conn.commit() + conn.close() + + result = worker._get_pending_count_by_job() + + assert result == {} + + +def test_get_pending_count_by_job_handles_db_error_gracefully(repair_worker_with_temp_db): + """Pin: any DB exception → returns ``{}``. Job-card badge falls + back to historical count via the ``or 0`` safety in JS.""" + worker, _ = repair_worker_with_temp_db + + # Replace _get_connection with one that raises. + def _raise(): + raise sqlite3.OperationalError("simulated") + worker.db._get_connection = _raise + + result = worker._get_pending_count_by_job() + + assert result == {} diff --git a/tests/test_discogs_collection_source.py b/tests/test_discogs_collection_source.py new file mode 100644 index 00000000..c037cd23 --- /dev/null +++ b/tests/test_discogs_collection_source.py @@ -0,0 +1,303 @@ +"""Tests for the Discogs collection source on Your Albums. + +Discord request (Jhones + BoulderBadgeDad): pull user's Discogs +collection into the Your Albums section on Discover, similar to how +Spotify Liked Albums works. Implementation adds Discogs as a fourth +source to the existing 3-source pipeline (Spotify / Tidal / Deezer) +with click-context dispatch so Discogs albums open with Discogs +release detail (vinyl/CD format info, year, label, tracklist). + +Tests pin: +- DiscogsClient.get_user_collection — pagination, response + normalization, disambiguation suffix stripping, missing-token + defensive return. +- DiscogsClient.get_release — passthrough to /releases/{id}. +- liked_albums_pool — discogs_release_id column round-trips through + the upsert + get path. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from core.discogs_client import DiscogsClient + + +# --------------------------------------------------------------------------- +# DiscogsClient.get_user_collection +# --------------------------------------------------------------------------- + + +@pytest.fixture +def authed_client(): + """A DiscogsClient with a fake token so is_authenticated() returns True + without hitting the real API.""" + return DiscogsClient(token='dummy_test_token') + + +def test_get_user_collection_returns_empty_without_token(monkeypatch): + """Defensive: no token → empty list, never raises. Discogs collection + is private so an unauthenticated call would 403 anyway. + + DiscogsClient's constructor falls back to ``config_manager.get( + 'discogs.token')`` when no token is passed — including when the + empty-string sentinel is passed (because empty-string is falsy). + Stub the config lookup so this test stays deterministic regardless + of the developer's local config (which may have a real token set + after using the Your Albums Discogs source feature).""" + from config.settings import config_manager + monkeypatch.setattr(config_manager, 'get', + lambda key, default=None: '' if key == 'discogs.token' else default) + client = DiscogsClient(token='') + assert client.get_user_collection() == [] + + +def test_get_user_collection_normalizes_response_shape(authed_client): + """Each release becomes the dict shape upsert_liked_album expects.""" + fake_response = { + 'pagination': {'pages': 1, 'page': 1}, + 'releases': [ + {'id': 12345, 'basic_information': { + 'title': 'GNX', + 'artists': [{'name': 'Kendrick Lamar'}], + 'cover_image': 'https://img.discogs.com/x.jpg', + 'year': 2024, + }}, + ], + } + + def _fake_get(endpoint, params=None): + if endpoint == '/oauth/identity': + return {'username': 'testuser'} + return fake_response + + with patch.object(authed_client, '_api_get', side_effect=_fake_get): + result = authed_client.get_user_collection() + + assert len(result) == 1 + r = result[0] + assert r['album_name'] == 'GNX' + assert r['artist_name'] == 'Kendrick Lamar' + assert r['release_id'] == 12345 + assert r['image_url'] == 'https://img.discogs.com/x.jpg' + assert r['release_date'] == '2024' + + +def test_get_user_collection_strips_discogs_disambiguation_suffix(authed_client): + """Discogs appends '(N)' to artist names when there are multiple + artists with the same name (e.g. 'Madonna (3)'). Strip it so the + name matches what Spotify/Tidal/Deezer use.""" + fake_response = { + 'pagination': {'pages': 1, 'page': 1}, + 'releases': [ + {'id': 1, 'basic_information': { + 'title': 'X', 'artists': [{'name': 'Madonna (3)'}], + 'cover_image': '', 'year': 2020, + }}, + ], + } + with patch.object(authed_client, '_api_get', + side_effect=lambda e, p=None: ({'username': 'u'} if e == '/oauth/identity' else fake_response)): + result = authed_client.get_user_collection() + + assert result[0]['artist_name'] == 'Madonna' + + +def test_get_user_collection_handles_missing_year(authed_client): + """Year 0 / missing → empty release_date string (NOT '0').""" + fake_response = { + 'pagination': {'pages': 1, 'page': 1}, + 'releases': [ + {'id': 1, 'basic_information': { + 'title': 'Album', + 'artists': [{'name': 'Artist'}], + 'cover_image': '', + 'year': 0, + }}, + ], + } + with patch.object(authed_client, '_api_get', + side_effect=lambda e, p=None: ({'username': 'u'} if e == '/oauth/identity' else fake_response)): + result = authed_client.get_user_collection() + + assert result[0]['release_date'] == '' + + +def test_get_user_collection_skips_releases_with_missing_required_fields(authed_client): + """Defensive: releases without title or artist are skipped, not crashed on.""" + fake_response = { + 'pagination': {'pages': 1, 'page': 1}, + 'releases': [ + {'id': 1, 'basic_information': {'title': 'Has Both', 'artists': [{'name': 'Artist'}]}}, + {'id': 2, 'basic_information': {'title': '', 'artists': [{'name': 'No Title'}]}}, + {'id': 3, 'basic_information': {'title': 'No Artists', 'artists': []}}, + ], + } + with patch.object(authed_client, '_api_get', + side_effect=lambda e, p=None: ({'username': 'u'} if e == '/oauth/identity' else fake_response)): + result = authed_client.get_user_collection() + + assert len(result) == 1 + assert result[0]['album_name'] == 'Has Both' + + +def test_get_user_collection_paginates(authed_client): + """Walks all pages until pagination.pages is reached.""" + page_responses = { + 1: {'pagination': {'pages': 2, 'page': 1}, + 'releases': [{'id': 1, 'basic_information': {'title': 'A', 'artists': [{'name': 'X'}]}}]}, + 2: {'pagination': {'pages': 2, 'page': 2}, + 'releases': [{'id': 2, 'basic_information': {'title': 'B', 'artists': [{'name': 'Y'}]}}]}, + } + call_count = {'n': 0} + + def _fake_get(endpoint, params=None): + if endpoint == '/oauth/identity': + return {'username': 'u'} + page = (params or {}).get('page', 1) + call_count['n'] += 1 + return page_responses.get(page) + + with patch.object(authed_client, '_api_get', side_effect=_fake_get): + result = authed_client.get_user_collection() + + assert len(result) == 2 + assert {r['release_id'] for r in result} == {1, 2} + + +def test_get_user_collection_caps_at_max_pages(authed_client): + """Guard against runaway pagination — stops after max_pages even if + the API claims more pages exist.""" + fake_response = { + 'pagination': {'pages': 9999, 'page': 1}, + 'releases': [{'id': 1, 'basic_information': {'title': 'A', 'artists': [{'name': 'X'}]}}], + } + with patch.object(authed_client, '_api_get', + side_effect=lambda e, p=None: ({'username': 'u'} if e == '/oauth/identity' else fake_response)): + # max_pages=2 — should request exactly 2 pages and stop + result = authed_client.get_user_collection(max_pages=2) + + # Each page returned 1 release — capped at 2 pages = 2 releases + assert len(result) == 2 + + +def test_get_user_collection_uses_explicit_username(authed_client): + """When username is passed explicitly, skip the /oauth/identity + lookup. Useful for callers that already know the username.""" + captured_endpoints = [] + + def _fake_get(endpoint, params=None): + captured_endpoints.append(endpoint) + return {'pagination': {'pages': 1, 'page': 1}, 'releases': []} + + with patch.object(authed_client, '_api_get', side_effect=_fake_get): + authed_client.get_user_collection(username='explicituser') + + # /oauth/identity should NOT have been called + assert '/oauth/identity' not in captured_endpoints + # Collection endpoint includes the explicit username + assert any('explicituser' in e for e in captured_endpoints) + + +# --------------------------------------------------------------------------- +# DiscogsClient.get_release +# --------------------------------------------------------------------------- + + +def test_get_release_passes_id_through_to_api(authed_client): + """Thin wrapper — confirm endpoint shape.""" + captured = [] + with patch.object(authed_client, '_api_get', + side_effect=lambda e, p=None: captured.append(e) or {'id': 999}): + result = authed_client.get_release(999) + assert captured == ['/releases/999'] + assert result == {'id': 999} + + +def test_get_release_returns_none_for_invalid_id(authed_client): + """Defensive: non-numeric / falsy id → None, no API call.""" + with patch.object(authed_client, '_api_get') as mock_api: + assert authed_client.get_release(None) is None + assert authed_client.get_release('not_a_number') is None + assert authed_client.get_release(0) is None + mock_api.assert_not_called() + + +# --------------------------------------------------------------------------- +# liked_albums_pool — discogs_release_id column +# --------------------------------------------------------------------------- + + +def test_liked_albums_discogs_release_id_roundtrip(): + """upsert with source_id_type='discogs' stores in discogs_release_id; + get_liked_albums returns it on the row.""" + from database.music_database import get_database + db = get_database() + + # Use a high profile_id to avoid colliding with real data + test_profile = 9991 + try: + ok = db.upsert_liked_album( + album_name='Test Disc Album', artist_name='Test Disc Artist', + source_service='discogs', + source_id='987654', source_id_type='discogs', + image_url=None, release_date='2023', total_tracks=10, + profile_id=test_profile, + ) + assert ok is True + + result = db.get_liked_albums(profile_id=test_profile, page=1, per_page=10) + assert result['total'] == 1 + row = result['albums'][0] + assert row['discogs_release_id'] == '987654' + assert row['album_name'] == 'Test Disc Album' + assert 'discogs' in row['source_services'] + finally: + # Clean up + conn = db._get_connection() + cur = conn.cursor() + cur.execute("DELETE FROM liked_albums_pool WHERE profile_id = ?", (test_profile,)) + conn.commit() + conn.close() + + +def test_liked_albums_multi_source_carries_both_ids(): + """If an album is added from Spotify AND from Discogs, both + spotify_album_id and discogs_release_id end up on the same row + via the dedup-by-normalized-key upsert.""" + from database.music_database import get_database + db = get_database() + + test_profile = 9992 + try: + # Add via Spotify first + db.upsert_liked_album( + album_name='Same Album', artist_name='Same Artist', + source_service='spotify', + source_id='spotify_id_xyz', source_id_type='spotify', + image_url=None, release_date='', total_tracks=0, + profile_id=test_profile, + ) + # Then add the same album via Discogs — should dedupe + db.upsert_liked_album( + album_name='Same Album', artist_name='Same Artist', + source_service='discogs', + source_id='discogs_id_999', source_id_type='discogs', + image_url=None, release_date='', total_tracks=0, + profile_id=test_profile, + ) + + result = db.get_liked_albums(profile_id=test_profile, page=1, per_page=10) + assert result['total'] == 1 # deduped to one row + row = result['albums'][0] + assert row['spotify_album_id'] == 'spotify_id_xyz' + assert row['discogs_release_id'] == 'discogs_id_999' + assert set(row['source_services']) == {'spotify', 'discogs'} + finally: + conn = db._get_connection() + cur = conn.cursor() + cur.execute("DELETE FROM liked_albums_pool WHERE profile_id = ?", (test_profile,)) + conn.commit() + conn.close() diff --git a/tests/test_download_orchestrator_soundcloud.py b/tests/test_download_orchestrator_soundcloud.py new file mode 100644 index 00000000..a3d1ab0a --- /dev/null +++ b/tests/test_download_orchestrator_soundcloud.py @@ -0,0 +1,247 @@ +"""Integration tests for SoundCloud wiring inside DownloadOrchestrator. + +The standalone SoundcloudClient is exhaustively unit-tested in +``tests/test_soundcloud_client.py``. These tests verify the *plumbing*: +the orchestrator constructs a SoundCloud client at startup, exposes it +via the same lookup APIs every other source uses, dispatches downloads +to it when the username matches, and includes it in the hybrid-mode +fan-out / status / cancel / clear paths. + +The intent is plug-and-play parity: any code that walks the +orchestrator's source list (UI, status endpoints, batch tracker) +picks up SoundCloud automatically without per-source special cases. +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import patch, MagicMock + +import pytest + +from core.download_orchestrator import DownloadOrchestrator +from core.soundcloud_client import SoundcloudClient + + +def _run(coro): + return asyncio.run(coro) + + +@pytest.fixture +def orchestrator() -> DownloadOrchestrator: + """Real orchestrator with real (but mostly idle) clients.""" + return DownloadOrchestrator() + + +# --------------------------------------------------------------------------- +# Construction +# --------------------------------------------------------------------------- + + +def test_orchestrator_constructs_soundcloud_client(orchestrator: DownloadOrchestrator) -> None: + assert orchestrator.client('soundcloud') is not None + assert isinstance(orchestrator.client('soundcloud'), SoundcloudClient) + + +def test_client_lookup_resolves_soundcloud(orchestrator: DownloadOrchestrator) -> None: + """Verify the registry-backed name → client lookup includes SoundCloud.""" + assert orchestrator.client('soundcloud') is orchestrator.registry.get('soundcloud') + + +def test_client_lookup_returns_none_for_unknown(orchestrator: DownloadOrchestrator) -> None: + """Sanity: unknown sources don't somehow resolve to SoundCloud.""" + assert orchestrator.client('made_up') is None + + +# --------------------------------------------------------------------------- +# Status surface +# --------------------------------------------------------------------------- + + +def test_get_source_status_includes_soundcloud(orchestrator: DownloadOrchestrator) -> None: + """Every other source has a key here; SoundCloud should too. The UI + walks this dict to render configured-status badges.""" + status = orchestrator.get_source_status() + assert 'soundcloud' in status + # yt-dlp is in requirements.txt → SoundCloud is configured by default + assert status['soundcloud'] is True + + +# --------------------------------------------------------------------------- +# Download dispatch +# --------------------------------------------------------------------------- + + +def test_download_routes_soundcloud_username_to_client(orchestrator: DownloadOrchestrator) -> None: + """The dispatcher must route ``username='soundcloud'`` to the SoundCloud + client, not to Soulseek (the default fallback path).""" + sentinel = 'sc-download-id-xyz' + + async def _fake_download(username, filename, file_size=0): + return sentinel + + with patch.object(orchestrator.client('soundcloud'), 'download', side_effect=_fake_download) as mock_dl: + result = _run(orchestrator.download( + 'soundcloud', + '999||https://soundcloud.com/x/y||Display', + file_size=0, + )) + assert result == sentinel + mock_dl.assert_called_once() + + +def test_download_unknown_username_still_falls_to_soulseek(orchestrator: DownloadOrchestrator) -> None: + """Adding SoundCloud must not change the legacy Soulseek-fallback + behavior for unrecognized usernames.""" + if orchestrator.client('soulseek') is None: + pytest.skip("Soulseek client unavailable in this environment") + + async def _fake_soulseek_download(username, filename, file_size=0): + return 'soulseek-id' + + with patch.object(orchestrator.client('soulseek'), 'download', side_effect=_fake_soulseek_download) as mock_dl: + result = _run(orchestrator.download('some_random_user', 'file.mp3', 0)) + assert result == 'soulseek-id' + mock_dl.assert_called_once() + + +# --------------------------------------------------------------------------- +# Hybrid mode +# --------------------------------------------------------------------------- + + +def test_hybrid_search_iterates_soundcloud_when_in_order(orchestrator: DownloadOrchestrator) -> None: + """When SoundCloud appears in the hybrid_order list, the orchestrator + must walk through its search results just like any other source.""" + orchestrator.mode = 'hybrid' + orchestrator.hybrid_order = ['soundcloud'] + + fake_track = MagicMock() + fake_track.username = 'soundcloud' + + async def _fake_search(query, timeout=None, progress_callback=None): + return ([fake_track], []) + + with patch.object(orchestrator.client('soundcloud'), 'search', side_effect=_fake_search), \ + patch.object(orchestrator.client('soundcloud'), 'is_configured', return_value=True): + tracks, albums = _run(orchestrator.search("any query")) + + assert tracks == [fake_track] + assert albums == [] + + +def test_hybrid_search_skips_unconfigured_soundcloud(orchestrator: DownloadOrchestrator) -> None: + """Defensive: if SoundCloud is unconfigured (yt-dlp missing), the + hybrid loop must skip it cleanly and continue to the next source.""" + orchestrator.mode = 'hybrid' + orchestrator.hybrid_order = ['soundcloud', 'soulseek'] + + if orchestrator.client('soulseek') is None: + pytest.skip("Soulseek client unavailable in this environment") + + soulseek_track = MagicMock() + soulseek_track.username = 'unrelated_user' + + async def _fake_soulseek_search(query, timeout=None, progress_callback=None): + return ([soulseek_track], []) + + with patch.object(orchestrator.client('soundcloud'), 'is_configured', return_value=False), \ + patch.object(orchestrator.client('soulseek'), 'is_configured', return_value=True), \ + patch.object(orchestrator.client('soulseek'), 'search', side_effect=_fake_soulseek_search): + tracks, _ = _run(orchestrator.search("any")) + + assert tracks == [soulseek_track] + + +# --------------------------------------------------------------------------- +# Aggregate operations +# --------------------------------------------------------------------------- + + +def test_get_all_downloads_walks_soundcloud(orchestrator: DownloadOrchestrator) -> None: + """Active-downloads endpoint pulls from every client; SoundCloud's + queue must show up in the aggregate.""" + fake_status = MagicMock(id='sc-1', filename='x', state='InProgress, Downloading') + + async def _fake_get_all(): + return [fake_status] + + with patch.object(orchestrator.client('soundcloud'), 'get_all_downloads', side_effect=_fake_get_all): + all_dl = _run(orchestrator.get_all_downloads()) + + assert any(d is fake_status for d in all_dl) + + +def test_get_download_status_finds_soundcloud_id(orchestrator: DownloadOrchestrator) -> None: + """Status lookup must check SoundCloud — orchestrator iterates every + client until one finds the id.""" + fake_status = MagicMock(id='sc-2') + + async def _fake_get_status(download_id): + return fake_status if download_id == 'sc-2' else None + + with patch.object(orchestrator.client('soundcloud'), 'get_download_status', side_effect=_fake_get_status): + result = _run(orchestrator.get_download_status('sc-2')) + + assert result is fake_status + + +def test_cancel_routes_soundcloud_username(orchestrator: DownloadOrchestrator) -> None: + """Username-routed cancel must dispatch to the SoundCloud client when + username='soundcloud' is provided.""" + async def _fake_cancel(download_id, username=None, remove=False): + return True + + with patch.object(orchestrator.client('soundcloud'), 'cancel_download', side_effect=_fake_cancel) as mock_cancel: + ok = _run(orchestrator.cancel_download('sc-3', username='soundcloud')) + assert ok is True + mock_cancel.assert_called_once() + + +def test_clear_completed_walks_soundcloud(orchestrator: DownloadOrchestrator) -> None: + """Bulk clear-all-completed must call SoundCloud's clear method. + + We assert SoundCloud got called — not that the overall result is + True, since other sibling clients in the same orchestrator may + return False for unrelated reasons (e.g. an unrelated client + throwing). The contract this test pins is "SoundCloud is included + in the iteration", which is what plug-and-play parity requires. + """ + async def _fake_clear(): + return True + + with patch.object(orchestrator.client('soundcloud'), 'clear_all_completed_downloads', side_effect=_fake_clear) as mock_clear: + _run(orchestrator.clear_all_completed_downloads()) + mock_clear.assert_called_once() + + +# --------------------------------------------------------------------------- +# Mode-only routing +# --------------------------------------------------------------------------- + + +def test_soundcloud_only_mode_uses_soundcloud(orchestrator: DownloadOrchestrator) -> None: + """When mode='soundcloud', search must be dispatched only to the + SoundCloud client — not soulseek or any other source.""" + orchestrator.mode = 'soundcloud' + + async def _fake_search(query, timeout=None, progress_callback=None): + return ([MagicMock(username='soundcloud')], []) + + with patch.object(orchestrator.client('soundcloud'), 'search', side_effect=_fake_search) as mock_sc, \ + patch.object(orchestrator.client('soulseek'), 'search', side_effect=AssertionError("soulseek must not be searched")): + tracks, _ = _run(orchestrator.search("any")) + + assert len(tracks) == 1 + mock_sc.assert_called_once() + + +def test_streaming_sources_tuple_includes_soundcloud() -> None: + """The validation/streaming-source tuples used to pick scoring + behavior must include SoundCloud — otherwise SoundCloud results + would skip the matching-engine validation in + search_and_download_best.""" + from core.downloads import validation + from inspect import getsource + src = getsource(validation.filter_streaming_results) if hasattr(validation, 'filter_streaming_results') else getsource(validation) + assert 'soundcloud' in src, "core.downloads.validation must include 'soundcloud' in _streaming_sources" diff --git a/tests/test_download_plugin_conformance.py b/tests/test_download_plugin_conformance.py new file mode 100644 index 00000000..2b0f4e9d --- /dev/null +++ b/tests/test_download_plugin_conformance.py @@ -0,0 +1,163 @@ +"""Pin the structural conformance of every download source plugin +class to ``DownloadSourcePlugin``. + +Each registered source class MUST: +- Implement every protocol method by name. +- Mark async methods as `async def` so the orchestrator can `await` + them uniformly. + +When someone adds a new source (e.g. Usenet) and forgets one of +these methods, this test fails at the contract — long before the +first real download attempt would have raised AttributeError in +production. When someone CHANGES the contract (adds a method to +the protocol), this test forces every existing source to be +updated. + +Catches the smell that motivated the refactor in the first place: +8 sources independently grew the same shape because every +consumer site needed the same calls, but nothing enforced parity. + +NOTE on test design: these tests check CLASSES, not instances. +Instantiating real client classes (TidalDownloadClient, etc.) at +fixture setup pollutes module-level state in tidalapi / spotipy +imports and breaks downstream tests that rely on a clean import +graph. Class-level checks are equally strict for structural +conformance — the protocol only constrains the method surface, not +runtime instance behavior. +""" + +from __future__ import annotations + +import inspect + +import pytest + + +REQUIRED_SYNC_METHODS = {'is_configured'} + +REQUIRED_ASYNC_METHODS = { + 'check_connection', + 'search', + 'download', + 'get_all_downloads', + 'get_download_status', + 'cancel_download', + 'clear_all_completed_downloads', +} + + +def _import_plugin_classes(): + """Import every download source class lazily inside the test + rather than at module load — avoids dragging tidalapi / + spotipy / yt-dlp imports into every other test module's + collection phase.""" + from core.soulseek_client import SoulseekClient + from core.youtube_client import YouTubeClient + from core.tidal_download_client import TidalDownloadClient + from core.qobuz_client import QobuzClient + from core.hifi_client import HiFiClient + from core.deezer_download_client import DeezerDownloadClient + from core.lidarr_download_client import LidarrDownloadClient + from core.soundcloud_client import SoundcloudClient + + return { + 'soulseek': SoulseekClient, + 'youtube': YouTubeClient, + 'tidal': TidalDownloadClient, + 'qobuz': QobuzClient, + 'hifi': HiFiClient, + 'deezer': DeezerDownloadClient, + 'lidarr': LidarrDownloadClient, + 'soundcloud': SoundcloudClient, + } + + +def test_default_registry_registers_all_eight_sources(): + """Smoke check that the foundation registry knows about every + source the orchestrator historically dispatched to. If someone + drops a registration here, every other test in this module would + silently miss the missing source.""" + from core.download_plugins.registry import build_default_registry + + registry = build_default_registry() + expected = { + 'soulseek', 'youtube', 'tidal', 'qobuz', + 'hifi', 'deezer', 'lidarr', 'soundcloud', + } + assert set(registry.names()) == expected + + +def test_deezer_dl_alias_is_registered_against_deezer_spec(): + """Legacy ``deezer_dl`` source-name string used in config + per- + source dispatch must keep resolving — frontend, settings, + download_orchestrator's username dispatch all depend on it.""" + from core.download_plugins.registry import build_default_registry + + registry = build_default_registry() + spec = registry.get_spec('deezer_dl') + assert spec is not None + assert spec.name == 'deezer' + assert 'deezer_dl' in spec.aliases + + +@pytest.mark.parametrize('plugin_name', [ + 'soulseek', 'youtube', 'tidal', 'qobuz', + 'hifi', 'deezer', 'lidarr', 'soundcloud', +]) +def test_plugin_class_has_all_required_methods(plugin_name): + """Every registered plugin class exposes every protocol method + by name. Diagnostic-friendly: tells you WHICH method is missing + when a new source is added without all the required methods.""" + classes = _import_plugin_classes() + cls = classes[plugin_name] + + missing = [] + for method_name in REQUIRED_SYNC_METHODS | REQUIRED_ASYNC_METHODS: + if not hasattr(cls, method_name): + missing.append(method_name) + assert not missing, ( + f"{plugin_name} ({cls.__name__}) missing methods: {missing}" + ) + + +@pytest.mark.parametrize('plugin_name', [ + 'soulseek', 'youtube', 'tidal', 'qobuz', + 'hifi', 'deezer', 'lidarr', 'soundcloud', +]) +def test_plugin_class_async_methods_are_coroutines(plugin_name): + """Methods declared async in the protocol must be async on every + plugin class. A sync `download()` would silently skip the + orchestrator's `await` and return a coroutine object instead of + a download_id — the kind of bug that only surfaces at runtime + against a live user.""" + classes = _import_plugin_classes() + cls = classes[plugin_name] + + not_async = [] + for method_name in REQUIRED_ASYNC_METHODS: + method = getattr(cls, method_name, None) + if method is None: + continue + if not inspect.iscoroutinefunction(method): + not_async.append(method_name) + assert not not_async, ( + f"{plugin_name} ({cls.__name__}) declared these methods as " + f"sync but the protocol requires async: {not_async}" + ) + + +def test_orchestrator_uses_registry_for_dispatch(): + """The orchestrator must hold a registry reference and the generic + ``client(name)`` accessor must return the same instances the + registry holds. Per-source attribute aliases (``orchestrator.soulseek`` + etc.) were removed in favor of ``orchestrator.client('soulseek')``; + the legacy alias name (``deezer_dl``) still resolves to the canonical + deezer plugin via the registry's alias map.""" + from core.download_orchestrator import DownloadOrchestrator + + orchestrator = DownloadOrchestrator() + assert hasattr(orchestrator, 'registry') + assert orchestrator.client('soulseek') is orchestrator.registry.get('soulseek') + assert orchestrator.client('youtube') is orchestrator.registry.get('youtube') + assert orchestrator.client('deezer_dl') is orchestrator.registry.get('deezer') + assert orchestrator.client('lidarr') is orchestrator.registry.get('lidarr') diff --git a/tests/test_duplicate_detector_slskd_dedup.py b/tests/test_duplicate_detector_slskd_dedup.py index 72b7da38..53de9f69 100644 --- a/tests/test_duplicate_detector_slskd_dedup.py +++ b/tests/test_duplicate_detector_slskd_dedup.py @@ -66,6 +66,8 @@ class _FakeContext: def _create_finding(self, **kwargs): self.findings.append(kwargs) + # Mirror real `_create_finding` contract: True on insert. + return True # --------------------------------------------------------------------------- diff --git a/tests/test_enrichment_services.py b/tests/test_enrichment_services.py new file mode 100644 index 00000000..b6f8bdb5 --- /dev/null +++ b/tests/test_enrichment_services.py @@ -0,0 +1,402 @@ +"""Tests for the enrichment service registry + generic Flask blueprint. + +Covers the registry contract (registration / lookup / fallback status +shape) and the generic ``/api/enrichment//...`` routes, +including per-service quirks (rate-limit pre-resume guard, auto-pause +token cleanup, persisted-pause config keys, extra default fields). +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +import pytest +from flask import Flask + +from core.enrichment.api import configure as configure_api, create_blueprint +from core.enrichment.services import ( + EnrichmentService, + all_service_ids, + all_services, + clear_registry, + get_service, + register_services, +) + + +# --------------------------------------------------------------------------- +# Test doubles +# --------------------------------------------------------------------------- + + +class _FakeWorker: + """Captures pause / resume calls + returns controllable get_stats.""" + + def __init__(self, stats: Dict[str, Any] | None = None): + self.stats = stats or { + 'enabled': True, 'running': True, 'paused': False, + 'current_item': None, + 'stats': {'matched': 5, 'not_found': 1, 'pending': 10, 'errors': 0}, + 'progress': {}, + } + self.pause_calls = 0 + self.resume_calls = 0 + self.pause_should_raise: Exception | None = None + self.resume_should_raise: Exception | None = None + + def pause(self) -> None: + if self.pause_should_raise: + raise self.pause_should_raise + self.pause_calls += 1 + + def resume(self) -> None: + if self.resume_should_raise: + raise self.resume_should_raise + self.resume_calls += 1 + + def get_stats(self) -> Dict[str, Any]: + return self.stats + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _reset_registry(): + """Every test starts from a clean registry.""" + clear_registry() + yield + clear_registry() + + +@pytest.fixture +def host_state(): + """Host-side state collections (config, auto-pause set, yield-override set).""" + state = { + 'config': {}, + 'auto_paused': set(), + 'yield_override': set(), + } + configure_api( + config_set=lambda k, v: state['config'].__setitem__(k, v), + auto_paused_discard=lambda token: state['auto_paused'].discard(token), + yield_override_add=lambda token: state['yield_override'].add(token), + ) + yield state + # Reset hooks so other tests run on a clean slate. + configure_api(config_set=None, auto_paused_discard=None, yield_override_add=None) + + +@pytest.fixture +def app(host_state): + """Flask app with the enrichment blueprint registered.""" + app = Flask(__name__) + app.register_blueprint(create_blueprint()) + return app + + +@pytest.fixture +def client(app): + return app.test_client() + + +# --------------------------------------------------------------------------- +# Registry behavior +# --------------------------------------------------------------------------- + + +class TestRegistry: + def test_register_and_lookup(self): + worker = _FakeWorker() + svc = EnrichmentService( + id='example', display_name='Example', worker_getter=lambda: worker, + ) + register_services([svc]) + assert get_service('example') is svc + assert all_service_ids() == ['example'] + assert all_services() == [svc] + + def test_unknown_service_returns_none(self): + register_services([]) + assert get_service('does_not_exist') is None + + def test_re_register_replaces(self): + register_services([ + EnrichmentService(id='a', display_name='A', worker_getter=lambda: None), + ]) + register_services([ + EnrichmentService(id='b', display_name='B', worker_getter=lambda: None), + ]) + assert get_service('a') is None + assert get_service('b') is not None + + def test_empty_id_rejected(self): + with pytest.raises(ValueError): + register_services([ + EnrichmentService(id='', display_name='X', worker_getter=lambda: None), + ]) + + def test_worker_getter_exception_returns_none(self): + def boom(): + raise RuntimeError("init failed") + + svc = EnrichmentService(id='broken', display_name='Broken', worker_getter=boom) + register_services([svc]) + assert svc.get_worker() is None + + def test_fallback_status_default_shape(self): + svc = EnrichmentService(id='x', display_name='X', worker_getter=lambda: None) + fb = svc.fallback_status() + assert fb['enabled'] is False + assert fb['running'] is False + assert fb['paused'] is False + assert fb['current_item'] is None + assert fb['stats'] == {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0} + assert fb['progress'] == {} + + def test_fallback_status_extra_defaults_merged(self): + """Tidal / Qobuz add ``'authenticated': False`` to the fallback.""" + svc = EnrichmentService( + id='tidal', display_name='Tidal', worker_getter=lambda: None, + extra_status_defaults={'authenticated': False}, + ) + fb = svc.fallback_status() + assert fb['authenticated'] is False + # And the standard keys still present. + assert fb['enabled'] is False + + def test_fallback_status_does_not_share_stats_dict(self): + svc = EnrichmentService(id='x', display_name='X', worker_getter=lambda: None) + fb1 = svc.fallback_status() + fb1['stats']['matched'] = 999 + fb2 = svc.fallback_status() + assert fb2['stats']['matched'] == 0 + + +# --------------------------------------------------------------------------- +# Status route +# --------------------------------------------------------------------------- + + +class TestStatusRoute: + def test_returns_worker_stats_when_initialized(self, client): + worker = _FakeWorker(stats={'enabled': True, 'matched': 42}) + register_services([ + EnrichmentService(id='spotify', display_name='Spotify', worker_getter=lambda: worker), + ]) + resp = client.get('/api/enrichment/spotify/status') + assert resp.status_code == 200 + assert resp.get_json() == {'enabled': True, 'matched': 42} + + def test_returns_fallback_when_worker_none(self, client): + register_services([ + EnrichmentService(id='spotify', display_name='Spotify', worker_getter=lambda: None), + ]) + resp = client.get('/api/enrichment/spotify/status') + assert resp.status_code == 200 + body = resp.get_json() + assert body['enabled'] is False + assert body['stats'] == {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0} + + def test_unknown_service_returns_404(self, client): + register_services([]) + resp = client.get('/api/enrichment/no_such_service/status') + assert resp.status_code == 404 + + def test_get_stats_exception_returns_500(self, client): + class BoomWorker: + def get_stats(self): + raise RuntimeError("db gone") + + register_services([ + EnrichmentService(id='x', display_name='X', worker_getter=lambda: BoomWorker()), + ]) + resp = client.get('/api/enrichment/x/status') + assert resp.status_code == 500 + assert 'db gone' in resp.get_json()['error'] + + +# --------------------------------------------------------------------------- +# Pause route +# --------------------------------------------------------------------------- + + +class TestPauseRoute: + def test_pause_calls_worker_and_persists_config(self, client, host_state): + worker = _FakeWorker() + register_services([ + EnrichmentService( + id='itunes', display_name='iTunes', worker_getter=lambda: worker, + config_paused_key='itunes_enrichment_paused', + ), + ]) + resp = client.post('/api/enrichment/itunes/pause') + assert resp.status_code == 200 + assert resp.get_json() == {'status': 'paused'} + assert worker.pause_calls == 1 + assert host_state['config']['itunes_enrichment_paused'] is True + + def test_pause_drops_auto_pause_token(self, client, host_state): + worker = _FakeWorker() + host_state['auto_paused'].add('lastfm-enrichment') + register_services([ + EnrichmentService( + id='lastfm', display_name='Last.fm', worker_getter=lambda: worker, + config_paused_key='lastfm_enrichment_paused', + auto_pause_token='lastfm-enrichment', + ), + ]) + resp = client.post('/api/enrichment/lastfm/pause') + assert resp.status_code == 200 + assert 'lastfm-enrichment' not in host_state['auto_paused'] + + def test_pause_without_config_key_skips_persistence(self, client, host_state): + worker = _FakeWorker() + register_services([ + EnrichmentService( + id='hydra', display_name='Hydra', worker_getter=lambda: worker, + config_paused_key='', # No persistence + ), + ]) + resp = client.post('/api/enrichment/hydra/pause') + assert resp.status_code == 200 + assert host_state['config'] == {} # Nothing persisted + + def test_pause_when_worker_none_returns_400(self, client): + register_services([ + EnrichmentService(id='x', display_name='X', worker_getter=lambda: None), + ]) + resp = client.post('/api/enrichment/x/pause') + assert resp.status_code == 400 + assert 'not initialized' in resp.get_json()['error'] + + def test_pause_worker_exception_returns_500(self, client): + worker = _FakeWorker() + worker.pause_should_raise = RuntimeError("worker dead") + register_services([ + EnrichmentService(id='x', display_name='X', worker_getter=lambda: worker), + ]) + resp = client.post('/api/enrichment/x/pause') + assert resp.status_code == 500 + + +# --------------------------------------------------------------------------- +# Resume route +# --------------------------------------------------------------------------- + + +class TestResumeRoute: + def test_resume_calls_worker_persists_and_adds_yield_override(self, client, host_state): + worker = _FakeWorker() + register_services([ + EnrichmentService( + id='spotify', display_name='Spotify', worker_getter=lambda: worker, + config_paused_key='spotify_enrichment_paused', + auto_pause_token='spotify-enrichment', + ), + ]) + resp = client.post('/api/enrichment/spotify/resume') + assert resp.status_code == 200 + assert resp.get_json() == {'status': 'running'} + assert worker.resume_calls == 1 + assert host_state['config']['spotify_enrichment_paused'] is False + assert 'spotify-enrichment' in host_state['yield_override'] + + def test_resume_blocked_by_pre_check_returns_429(self, client): + """Spotify rate-limit guard: pre-check returns (429, message).""" + worker = _FakeWorker() + register_services([ + EnrichmentService( + id='spotify', display_name='Spotify', worker_getter=lambda: worker, + config_paused_key='spotify_enrichment_paused', + pre_resume_check=lambda: (429, 'Cannot resume while Spotify is rate limited'), + ), + ]) + resp = client.post('/api/enrichment/spotify/resume') + assert resp.status_code == 429 + body = resp.get_json() + assert body['rate_limited'] is True + assert 'rate limited' in body['error'] + assert worker.resume_calls == 0 # Worker not touched + + def test_resume_pre_check_returning_none_passes(self, client): + worker = _FakeWorker() + register_services([ + EnrichmentService( + id='spotify', display_name='Spotify', worker_getter=lambda: worker, + pre_resume_check=lambda: None, + ), + ]) + resp = client.post('/api/enrichment/spotify/resume') + assert resp.status_code == 200 + assert worker.resume_calls == 1 + + def test_resume_pre_check_exception_treated_as_pass(self, client): + """A buggy pre-check shouldn't permanently lock out resume.""" + worker = _FakeWorker() + + def boom(): + raise RuntimeError("pre-check broke") + + register_services([ + EnrichmentService( + id='spotify', display_name='Spotify', worker_getter=lambda: worker, + pre_resume_check=boom, + ), + ]) + resp = client.post('/api/enrichment/spotify/resume') + assert resp.status_code == 200 + assert worker.resume_calls == 1 + + def test_resume_when_worker_none_returns_400(self, client): + register_services([ + EnrichmentService(id='x', display_name='X', worker_getter=lambda: None), + ]) + resp = client.post('/api/enrichment/x/resume') + assert resp.status_code == 400 + + def test_resume_worker_exception_returns_500(self, client): + worker = _FakeWorker() + worker.resume_should_raise = RuntimeError("worker dead") + register_services([ + EnrichmentService(id='x', display_name='X', worker_getter=lambda: worker), + ]) + resp = client.post('/api/enrichment/x/resume') + assert resp.status_code == 500 + + def test_resume_without_auto_pause_token_skips_yield_override(self, client, host_state): + """Services without an auto_pause_token (e.g. iTunes, Deezer) should + NOT add to yield_override — that's a Spotify/LastFM/Genius-only + mechanism.""" + worker = _FakeWorker() + register_services([ + EnrichmentService( + id='itunes', display_name='iTunes', worker_getter=lambda: worker, + config_paused_key='itunes_enrichment_paused', + auto_pause_token=None, + ), + ]) + resp = client.post('/api/enrichment/itunes/resume') + assert resp.status_code == 200 + assert host_state['yield_override'] == set() + + +# --------------------------------------------------------------------------- +# 404 path +# --------------------------------------------------------------------------- + + +class TestUnknownService: + @pytest.mark.parametrize('verb,path', [ + ('get', '/api/enrichment/no_such/status'), + ('post', '/api/enrichment/no_such/pause'), + ('post', '/api/enrichment/no_such/resume'), + ]) + def test_404_for_unknown_service(self, client, verb, path): + register_services([]) + method = getattr(client, verb) + resp = method(path) + assert resp.status_code == 404 + assert 'no_such' in resp.get_json()['error'] diff --git a/tests/test_hifi_instance_methods.py b/tests/test_hifi_instance_methods.py index d300591c..99574582 100644 --- a/tests/test_hifi_instance_methods.py +++ b/tests/test_hifi_instance_methods.py @@ -306,13 +306,17 @@ def test_seed_hifi_instances_does_not_duplicate_on_reseed(db): assert len(rows) == 1 -# ── error propagation ──────────────────────────────────────────────────── -# These methods now let DB errors bubble up so the route layer turns them -# into a 500 — the user sees a real failure instead of a phantom empty state. +# ── lazy-create recovery (issue #503) ──────────────────────────────────── +# When the bulk ``_initialize_database`` rolls back due to a brittle +# migration step, ``hifi_instances`` would normally be missing on the +# user's DB. The CRUD methods now ensure the table exists immediately +# before operating, so the operation succeeds rather than throwing +# "no such table: hifi_instances". def _db_without_hifi_table(): - """Returns a MusicDatabase with NO hifi_instances table.""" + """Returns a MusicDatabase with NO hifi_instances table — simulates + the broken-init state where ``_initialize_database`` rolled back.""" conn = sqlite3.connect(":memory:") conn.row_factory = sqlite3.Row @@ -326,43 +330,78 @@ def _db_without_hifi_table(): return _NoTableDB() -def test_get_hifi_instances_propagates_db_errors(): +def _hifi_table_exists(db): + cur = db._conn.cursor() + cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='hifi_instances'") + return cur.fetchone() is not None + + +def test_add_hifi_instance_creates_table_when_missing(): + """Pin issue #503 fix: add_hifi_instance must lazily create the + table when init failed to land it. Pre-fix this raised + ``no such table: hifi_instances`` and the user couldn't add any + HiFi instances at all.""" db = _db_without_hifi_table() - with pytest.raises(sqlite3.OperationalError): - db.get_hifi_instances() + assert not _hifi_table_exists(db) + + result = db.add_hifi_instance("http://recovery.com", priority=0) + + assert result is True + assert _hifi_table_exists(db) + rows = db.get_all_hifi_instances() + assert len(rows) == 1 + assert rows[0]["url"] == "http://recovery.com" -def test_get_all_hifi_instances_propagates_db_errors(): +def test_get_hifi_instances_creates_table_when_missing(): + """Empty result instead of OperationalError — read methods + self-heal too.""" db = _db_without_hifi_table() - with pytest.raises(sqlite3.OperationalError): - db.get_all_hifi_instances() + assert not _hifi_table_exists(db) + + rows = db.get_hifi_instances() + + assert rows == [] + assert _hifi_table_exists(db) -def test_add_hifi_instance_propagates_db_errors(): +def test_get_all_hifi_instances_creates_table_when_missing(): db = _db_without_hifi_table() - with pytest.raises(sqlite3.OperationalError): - db.add_hifi_instance("http://x.com") + rows = db.get_all_hifi_instances() + assert rows == [] + assert _hifi_table_exists(db) -def test_remove_hifi_instance_propagates_db_errors(): +def test_remove_hifi_instance_creates_table_when_missing(): + """Removing from a missing table is a no-op (returns False) — + doesn't crash.""" db = _db_without_hifi_table() - with pytest.raises(sqlite3.OperationalError): - db.remove_hifi_instance("http://x.com") + result = db.remove_hifi_instance("http://x.com") + assert result is False + assert _hifi_table_exists(db) -def test_toggle_hifi_instance_propagates_db_errors(): +def test_toggle_hifi_instance_creates_table_when_missing(): db = _db_without_hifi_table() - with pytest.raises(sqlite3.OperationalError): - db.toggle_hifi_instance("http://x.com", enabled=True) + result = db.toggle_hifi_instance("http://x.com", enabled=True) + assert result is False # Nothing to toggle + assert _hifi_table_exists(db) -def test_reorder_hifi_instances_propagates_db_errors(): +def test_reorder_hifi_instances_creates_table_when_missing(): + """Reorder with missing entries returns False (the existing + contract — caller-supplied URLs not present).""" db = _db_without_hifi_table() - with pytest.raises(sqlite3.OperationalError): - db.reorder_hifi_instances(["http://x.com"]) + result = db.reorder_hifi_instances(["http://x.com"]) + assert result is False # No matching rows + assert _hifi_table_exists(db) -def test_seed_hifi_instances_propagates_db_errors(): +def test_seed_hifi_instances_creates_table_when_missing(): + """Seeding into a missing table works — table is created, then + defaults inserted.""" db = _db_without_hifi_table() - with pytest.raises(sqlite3.OperationalError): - db.seed_hifi_instances(["http://x.com"]) + db.seed_hifi_instances(["http://default-a.com", "http://default-b.com"]) + rows = db.get_all_hifi_instances() + assert len(rows) == 2 + assert _hifi_table_exists(db) diff --git a/tests/test_integrity_failure_marks_task_failed.py b/tests/test_integrity_failure_marks_task_failed.py new file mode 100644 index 00000000..e91fe8d6 --- /dev/null +++ b/tests/test_integrity_failure_marks_task_failed.py @@ -0,0 +1,202 @@ +"""Pin the contract: integrity rejection must mark the task as failed. + +User report (Mr. Morale download): three tracks (Rich Interlude, +Savior Interlude, Savior) showed ✅ Completed in the modal but were +missing from disk. Log trace at line 932 of `core/imports/pipeline.py` +revealed the bug: + + No _final_processed_path in context for task — cannot verify, assuming success + +Inner ``post_process_matched_download`` quarantined the source file +(integrity check rejected duration mismatch on a wrong-content file), +which left no ``_final_processed_path`` in the context. The outer +verification wrapper saw no path and fell through to the "assuming +success" branch, marking the task as ✅ Completed even though the file +was in quarantine and would never reach the destination. + +Fix: the wrapper now explicitly checks for ``_integrity_failure_msg`` +and ``_race_guard_failed`` markers BEFORE the "assume success" branch. +If any failure marker is set, the task is marked failed with a +descriptive error message and the batch tracker is notified with +``success=False``. +""" + +from __future__ import annotations + +import threading +import types +from unittest.mock import patch + +import pytest + +import core.imports.pipeline as import_pipeline +import core.runtime_state as runtime_state + + +# --------------------------------------------------------------------------- +# Test scaffolding +# --------------------------------------------------------------------------- + + +@pytest.fixture +def _isolate_state(): + """Snapshot + restore the global runtime maps so this test can mutate + them without polluting other tests.""" + snapshot = { + 'tasks': dict(runtime_state.download_tasks), + 'batches': dict(runtime_state.download_batches), + 'matched_ctx': dict(runtime_state.matched_downloads_context), + } + runtime_state.download_tasks.clear() + runtime_state.download_batches.clear() + runtime_state.matched_downloads_context.clear() + yield + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(snapshot['tasks']) + runtime_state.download_batches.clear() + runtime_state.download_batches.update(snapshot['batches']) + runtime_state.matched_downloads_context.clear() + runtime_state.matched_downloads_context.update(snapshot['matched_ctx']) + + +def _build_runtime(completion_calls): + return types.SimpleNamespace( + automation_engine=None, + on_download_completed=lambda batch, task, success: completion_calls.append( + (batch, task, success) + ), + web_scan_manager=None, + repair_worker=None, + ) + + +def _seed_task(task_id: str = 't1', batch_id: str = 'b1') -> None: + runtime_state.download_tasks[task_id] = { + 'task_id': task_id, + 'batch_id': batch_id, + 'status': 'downloading', + 'track_info': {'name': 'Rich (Interlude)'}, + } + + +# --------------------------------------------------------------------------- +# The wrapper-level fix +# --------------------------------------------------------------------------- + + +def test_integrity_failure_marker_marks_task_failed(_isolate_state): + """When inner code sets ``_integrity_failure_msg``, the wrapper + must mark the task failed — NOT fall through to "assume success".""" + completion_calls = [] + runtime = _build_runtime(completion_calls) + + _seed_task('t1', 'b1') + + context = { + 'task_id': 't1', + 'batch_id': 'b1', + 'context_key': 'test::ctx', + # Simulate inner code's integrity-rejection state — file went to + # quarantine, _final_processed_path NEVER got set. + '_integrity_failure_msg': 'Duration mismatch: file is 163s, expected 152s (drift 11s)', + } + + # Inner post-processor is a no-op for this test — we're verifying the + # wrapper-level state machine. Stub everything inside `with_verification` + # that would otherwise touch real disk / acoustid / etc. + with patch.object(import_pipeline, 'post_process_matched_download', + lambda *a, **kw: None): + import_pipeline.post_process_matched_download_with_verification( + 'test::ctx', context, '/fake/source.flac', 't1', 'b1', runtime, + ) + + # Task explicitly marked failed with the integrity error message + assert runtime_state.download_tasks['t1']['status'] == 'failed' + assert 'integrity' in runtime_state.download_tasks['t1']['error_message'].lower() + # Batch tracker notified with success=False + assert ('b1', 't1', False) in completion_calls + # Did NOT fall through to "assume success" + assert ('b1', 't1', True) not in completion_calls + + +def test_race_guard_failure_marker_marks_task_failed(_isolate_state): + """Same contract for the race-guard-failed marker (source file + disappeared with no known destination).""" + completion_calls = [] + runtime = _build_runtime(completion_calls) + + _seed_task('t2', 'b2') + + context = { + 'task_id': 't2', + 'batch_id': 'b2', + 'context_key': 'test::ctx2', + '_race_guard_failed': True, + } + + with patch.object(import_pipeline, 'post_process_matched_download', + lambda *a, **kw: None): + import_pipeline.post_process_matched_download_with_verification( + 'test::ctx2', context, '/fake/source.flac', 't2', 'b2', runtime, + ) + + assert runtime_state.download_tasks['t2']['status'] == 'failed' + assert ('b2', 't2', False) in completion_calls + + +def test_no_failure_markers_still_assumes_success(_isolate_state): + """The pre-existing "assume success" fallback must STILL fire when + no failure markers are set — some legitimate flows complete without + setting `_final_processed_path`. Don't regress that behavior.""" + completion_calls = [] + runtime = _build_runtime(completion_calls) + + _seed_task('t3', 'b3') + + context = { + 'task_id': 't3', + 'batch_id': 'b3', + 'context_key': 'test::ctx3', + # No failure markers, no _final_processed_path + } + + with patch.object(import_pipeline, 'post_process_matched_download', + lambda *a, **kw: None), \ + patch.object(import_pipeline, '_mark_task_completed', + lambda task_id, ti: runtime_state.download_tasks[task_id].update( + {'status': 'completed'} + )): + import_pipeline.post_process_matched_download_with_verification( + 'test::ctx3', context, '/fake/source.flac', 't3', 'b3', runtime, + ) + + assert runtime_state.download_tasks['t3']['status'] == 'completed' + assert ('b3', 't3', True) in completion_calls + + +def test_integrity_failure_takes_priority_over_missing_final_path(_isolate_state): + """Integrity failure check must run BEFORE the missing-final-path + fallback. Both conditions are true (no final path AND integrity + failed); the failure wins.""" + completion_calls = [] + runtime = _build_runtime(completion_calls) + + _seed_task('t4', 'b4') + + context = { + 'task_id': 't4', + 'batch_id': 'b4', + 'context_key': 'test::ctx4', + '_integrity_failure_msg': 'duration mismatch', + # no _final_processed_path — would otherwise hit "assume success" + } + + with patch.object(import_pipeline, 'post_process_matched_download', + lambda *a, **kw: None): + import_pipeline.post_process_matched_download_with_verification( + 'test::ctx4', context, '/fake/source.flac', 't4', 'b4', runtime, + ) + + assert runtime_state.download_tasks['t4']['status'] == 'failed' + # Critical: must NOT have notified success + assert ('b4', 't4', True) not in completion_calls diff --git a/tests/test_library_disk_usage.py b/tests/test_library_disk_usage.py new file mode 100644 index 00000000..00707aa7 --- /dev/null +++ b/tests/test_library_disk_usage.py @@ -0,0 +1,282 @@ +"""Tests for the Library Disk Usage stat. + +Discord request (Samuel [KC]): show how much disk space the library +takes on the System Statistics page. Implementation piggybacks on the +existing deep scan — Plex/Jellyfin/Navidrome all return file size in +their track API responses, so we read it during the deep scan and +aggregate via SQL on demand. No filesystem walk involved. + +Tests pin: +- Schema migration is idempotent and backward-compatible (existing + rows get NULL file_size; new column doesn't break old inserts). +- Aggregator returns the empty-shape dict for fresh installs and + walks/sums correctly when populated. +- Per-format breakdown handles mixed extensions correctly. +- Defensive: empty / NULL / malformed paths don't crash. +""" + +from __future__ import annotations + +import os +import sqlite3 +import tempfile +import uuid +from pathlib import Path + +import pytest + +from database.music_database import MusicDatabase + + +@pytest.fixture +def db(tmp_path: Path) -> MusicDatabase: + """Build a fresh isolated MusicDatabase against a temp file.""" + db_path = tmp_path / 'test_library_size.db' + return MusicDatabase(database_path=str(db_path)) + + +def _insert_track(db: MusicDatabase, *, track_id: str, file_path: str, + file_size, album_id: str = 'a1', artist_id: str = 'ar1') -> None: + """Helper: seed an artist+album+track row with the given size.""" + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT OR IGNORE INTO artists (id, name) VALUES (?, ?)", + (artist_id, 'Test Artist')) + cur.execute("INSERT OR IGNORE INTO albums (id, artist_id, title) VALUES (?, ?, ?)", + (album_id, artist_id, 'Test Album')) + cur.execute( + "INSERT INTO tracks (id, album_id, artist_id, title, file_path, file_size) " + "VALUES (?, ?, ?, ?, ?, ?)", + (track_id, album_id, artist_id, f'track-{track_id}', file_path, file_size), + ) + conn.commit() + conn.close() + + +# --------------------------------------------------------------------------- +# Schema migration +# --------------------------------------------------------------------------- + + +def test_file_size_column_exists_after_init(db: MusicDatabase) -> None: + """Fresh install should have the column from the canonical + CREATE TABLE.""" + conn = db._get_connection() + cur = conn.cursor() + cur.execute("PRAGMA table_info(tracks)") + cols = {row[1] for row in cur.fetchall()} + conn.close() + assert 'file_size' in cols + + +def test_existing_tracks_have_null_file_size_after_migration(db: MusicDatabase) -> None: + """Backward-compat: rows inserted via the OLD schema (no file_size) + must still be readable, and querying file_size returns NULL — not + an error. Simulated by inserting a track without specifying + file_size (relies on column default = NULL).""" + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT OR IGNORE INTO artists (id, name) VALUES ('ar1', 'A')") + cur.execute("INSERT OR IGNORE INTO albums (id, artist_id, title) VALUES ('a1', 'ar1', 'Al')") + # Note: NOT specifying file_size — should default to NULL + cur.execute( + "INSERT INTO tracks (id, album_id, artist_id, title, file_path) " + "VALUES ('legacy_t', 'a1', 'ar1', 'L', '/x/legacy.flac')" + ) + conn.commit() + cur.execute("SELECT file_size FROM tracks WHERE id = 'legacy_t'") + row = cur.fetchone() + conn.close() + # Could be sqlite3.Row or tuple; both index by 0 + assert row[0] is None + + +# --------------------------------------------------------------------------- +# Aggregator +# --------------------------------------------------------------------------- + + +def test_aggregator_returns_empty_shape_for_fresh_install(db: MusicDatabase) -> None: + """No tracks inserted → has_data=False, total=0, no formats.""" + result = db.get_library_disk_usage() + assert result == { + 'total_bytes': 0, + 'tracks_with_size': 0, + 'tracks_without_size': 0, + 'by_format': {}, + 'has_data': False, + } + + +def test_aggregator_sums_known_sizes(db: MusicDatabase) -> None: + _insert_track(db, track_id='t1', file_path='/x/song1.flac', file_size=10_000_000) + _insert_track(db, track_id='t2', file_path='/x/song2.flac', file_size=5_000_000) + _insert_track(db, track_id='t3', file_path='/x/song3.mp3', file_size=3_000_000) + + result = db.get_library_disk_usage() + assert result['total_bytes'] == 18_000_000 + assert result['tracks_with_size'] == 3 + assert result['tracks_without_size'] == 0 + assert result['has_data'] is True + + +def test_aggregator_excludes_null_sizes_from_sum(db: MusicDatabase) -> None: + """Tracks without size are counted but don't contribute to total_bytes.""" + _insert_track(db, track_id='t1', file_path='/x/sized.flac', file_size=10_000_000) + _insert_track(db, track_id='t2', file_path='/x/null.flac', file_size=None) + + result = db.get_library_disk_usage() + assert result['total_bytes'] == 10_000_000 + assert result['tracks_with_size'] == 1 + assert result['tracks_without_size'] == 1 + # Has data — at least one track was measured + assert result['has_data'] is True + + +def test_aggregator_per_format_breakdown(db: MusicDatabase) -> None: + _insert_track(db, track_id='t1', file_path='/x/song.flac', file_size=10_000_000) + _insert_track(db, track_id='t2', file_path='/x/other.flac', file_size=5_000_000) + _insert_track(db, track_id='t3', file_path='/x/song.mp3', file_size=3_000_000) + _insert_track(db, track_id='t4', file_path='/x/song.m4a', file_size=2_000_000) + + result = db.get_library_disk_usage() + assert result['by_format'] == { + 'flac': 15_000_000, + 'mp3': 3_000_000, + 'm4a': 2_000_000, + } + + +def test_aggregator_handles_mixed_case_extensions(db: MusicDatabase) -> None: + """Extensions get lowercased so .FLAC and .flac group together.""" + _insert_track(db, track_id='t1', file_path='/x/song.FLAC', file_size=5_000_000) + _insert_track(db, track_id='t2', file_path='/x/other.flac', file_size=5_000_000) + + result = db.get_library_disk_usage() + assert result['by_format'] == {'flac': 10_000_000} + + +def test_aggregator_handles_paths_with_dots_in_album_name(db: MusicDatabase) -> None: + """Albums like 'M.A.A.D City' have dots in the path. Extension + extraction must use the LAST dot, not the first.""" + _insert_track( + db, track_id='t1', + file_path='/music/Kendrick Lamar/M.A.A.D City/01 - track.flac', + file_size=10_000_000, + ) + result = db.get_library_disk_usage() + assert result['by_format'] == {'flac': 10_000_000} + + +def test_aggregator_skips_paths_without_extension(db: MusicDatabase) -> None: + """Defensive: files without an extension don't show up in + by_format (would otherwise produce an empty-string key or junk).""" + _insert_track(db, track_id='t1', file_path='/x/no_extension', file_size=5_000_000) + _insert_track(db, track_id='t2', file_path='/x/song.flac', file_size=10_000_000) + + result = db.get_library_disk_usage() + assert result['total_bytes'] == 15_000_000 + assert result['by_format'] == {'flac': 10_000_000} + assert '' not in result['by_format'] + + +def test_aggregator_skips_empty_file_path(db: MusicDatabase) -> None: + """Empty string file_path → shouldn't appear in by_format.""" + _insert_track(db, track_id='t1', file_path='', file_size=5_000_000) + _insert_track(db, track_id='t2', file_path='/x/song.flac', file_size=10_000_000) + + result = db.get_library_disk_usage() + # Total still includes the empty-path track (it was measured) + assert result['total_bytes'] == 15_000_000 + # But by_format only has the one with a real extension + assert result['by_format'] == {'flac': 10_000_000} + + +def test_aggregator_skips_implausibly_long_extension(db: MusicDatabase) -> None: + """Extensions over 6 chars are filtered (would be junk from an + unusual filename like 'song.somethingweird').""" + _insert_track(db, track_id='t1', file_path='/x/song.somethingweird', file_size=5_000_000) + _insert_track(db, track_id='t2', file_path='/x/song.flac', file_size=10_000_000) + + result = db.get_library_disk_usage() + assert result['by_format'] == {'flac': 10_000_000} + + +# --------------------------------------------------------------------------- +# Backward compatibility — schema column ordering / NULL writes +# --------------------------------------------------------------------------- + + +def test_insert_or_update_media_track_persists_size_for_object_with_file_size(db: MusicDatabase) -> None: + """The Jellyfin/Navidrome/SoulSync track wrappers expose + `track_obj.file_size`. Verify insert_or_update_media_track reads + it and persists to the new column.""" + + class _FakeTrack: + def __init__(self): + self.ratingKey = 'fake_track_id_1' + self.title = 'Test Track' + self.trackNumber = 1 + self.duration = 200000 + self.path = '/library/Artist/Album/01 - track.flac' + self.bitRate = 1411 + self.file_size = 42_000_000 + + # Seed parent rows so FK constraints are satisfied + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT OR IGNORE INTO artists (id, name) VALUES ('ar2', 'Artist')") + cur.execute("INSERT OR IGNORE INTO albums (id, artist_id, title) VALUES ('al2', 'ar2', 'Album')") + conn.commit() + conn.close() + + db.insert_or_update_media_track(_FakeTrack(), album_id='al2', artist_id='ar2', + server_source='jellyfin') + + conn = db._get_connection() + cur = conn.cursor() + cur.execute("SELECT file_size FROM tracks WHERE id = 'fake_track_id_1'") + row = cur.fetchone() + conn.close() + assert row[0] == 42_000_000 + + +def test_insert_or_update_media_track_preserves_size_on_null_re_sync(db: MusicDatabase) -> None: + """If a subsequent deep scan returns no file_size for a track that + previously had one (e.g. server hiccup, rare Jellyfin response), + the COALESCE on UPDATE preserves the existing value rather than + blanking it. Pin the regression — losing data on every scan would + be worse than the original problem.""" + + class _FakeTrack: + def __init__(self, size): + self.ratingKey = 'fake_track_id_2' + self.title = 'Test' + self.trackNumber = 1 + self.duration = 200000 + self.path = '/library/Artist/Album/02 - track.flac' + self.bitRate = 1411 + self.file_size = size + + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT OR IGNORE INTO artists (id, name) VALUES ('ar3', 'Artist')") + cur.execute("INSERT OR IGNORE INTO albums (id, artist_id, title) VALUES ('al3', 'ar3', 'Album')") + conn.commit() + conn.close() + + # First sync — server reports 30 MB + db.insert_or_update_media_track(_FakeTrack(size=30_000_000), album_id='al3', + artist_id='ar3', server_source='jellyfin') + + # Second sync — server reports None (didn't include Size in MediaSources this time) + db.insert_or_update_media_track(_FakeTrack(size=None), album_id='al3', + artist_id='ar3', server_source='jellyfin') + + conn = db._get_connection() + cur = conn.cursor() + cur.execute("SELECT file_size FROM tracks WHERE id = 'fake_track_id_2'") + row = cur.fetchone() + conn.close() + # Original size preserved + assert row[0] == 30_000_000 diff --git a/tests/test_library_reorganize.py b/tests/test_library_reorganize.py index 50f97fa9..6c34c285 100644 --- a/tests/test_library_reorganize.py +++ b/tests/test_library_reorganize.py @@ -1,8 +1,29 @@ +"""Tests for the rewritten Library Reorganize repair job. + +Issue #500: pre-rewrite the job had its own tag-reading + transfer- +folder-grouping + template-application implementation. The +``is_album = group_size > 1`` heuristic misclassified album tracks +as singles when only one track of an album sat in the transfer folder +or when album tags varied across tracks. + +Post-rewrite the job delegates to the per-album planner +(``core.library_reorganize.preview_album_reorganize`` / +``reorganize_queue``) — no second move/template implementation. These +tests pin the delegation contract so future drift fails here instead +of at runtime against a real library. +""" + +from __future__ import annotations + import sys import types from types import SimpleNamespace +from unittest.mock import MagicMock -# Stub optional Spotify dependency so metadata_service can import in tests. +import pytest + + +# ── stubs (same shape used elsewhere in the suite) ────────────────────── if 'spotipy' not in sys.modules: spotipy = types.ModuleType('spotipy') oauth2 = types.ModuleType('spotipy.oauth2') @@ -36,97 +57,459 @@ if 'config.settings' not in sys.modules: sys.modules['config'] = config_mod sys.modules['config.settings'] = settings_mod -from core.repair_jobs import library_reorganize as lr + +from core.repair_jobs.library_reorganize import LibraryReorganizeJob +from core.repair_jobs.base import JobContext -class _FakeSearchClient: - def __init__(self, source_name, year=None): - self.source_name = source_name - self.year = year - self.calls = [] - - def search_albums(self, query, limit=3): - self.calls.append((query, limit)) - if self.year is None: - return [] - return [SimpleNamespace(release_date=f"{self.year}-01-01")] +# ── fixtures ────────────────────────────────────────────────────────── -def test_lookup_years_prefers_primary_source(monkeypatch): - deezer_client = _FakeSearchClient('deezer', '2022') - spotify_client = _FakeSearchClient('spotify', '1999') +class _FakeConfigManager: + """Minimal config manager. Reads via ``get(key, default)``.""" - monkeypatch.setattr(lr, 'get_primary_source', lambda: 'deezer') - monkeypatch.setattr(lr, 'get_source_priority', lambda primary: [primary, 'itunes', 'spotify']) - monkeypatch.setattr( - lr, - 'get_client_for_source', - lambda source: {'deezer': deezer_client, 'spotify': spotify_client}.get(source), + def __init__(self, settings: dict | None = None, file_org_enabled: bool = True, + active_server: str = 'plex'): + self._settings = settings or {} + self._file_org_enabled = file_org_enabled + self._active_server = active_server + + def get(self, key, default=None): + if key == 'file_organization.enabled': + return self._file_org_enabled + return self._settings.get(key, default) + + def get_active_media_server(self): + return self._active_server + + +class _FakeDB: + """Stand-in for MusicDatabase. Returns the canned album list from + ``_load_albums``'s SELECT. + + Supports per-server filtering: pass ``rows_by_server`` as a dict to + return different album sets depending on the SQL parameters. The + helper inspects the SQL string for ``server_source = ?`` and returns + the matching slice. Falls back to ``album_rows`` when ``rows_by_server`` + isn't set (back-compat with single-server tests).""" + + def __init__(self, album_rows: list = None, rows_by_server: dict = None): + self._album_rows = album_rows or [] + self._rows_by_server = rows_by_server or {} + + def _get_connection(self): + cursor = MagicMock() + rows_by_server = self._rows_by_server + default_rows = self._album_rows + + def _execute(sql, params=()): + if rows_by_server and 'server_source = ?' in sql and params: + cursor._captured_rows = rows_by_server.get(params[0], []) + else: + cursor._captured_rows = default_rows + cursor.fetchall.return_value = cursor._captured_rows + cursor.fetchone.return_value = (len(cursor._captured_rows),) + + cursor.execute.side_effect = _execute + cursor.fetchall.return_value = default_rows + cursor.fetchone.return_value = (len(default_rows),) + conn = MagicMock() + conn.cursor.return_value = cursor + return conn + + +@pytest.fixture +def make_context(): + """Build a JobContext with optional finding-collector.""" + + def _make(*, db, cm=None, dry_run=True, transfer='/tmp/transfer'): + findings = [] + + def _create_finding(**kwargs): + findings.append(kwargs) + return True # 'inserted' return value + + ctx = JobContext( + db=db, + transfer_folder=transfer, + config_manager=cm or _FakeConfigManager( + settings={ + f'repair.jobs.library_reorganize.settings.dry_run': dry_run, + }, + ), + create_finding=_create_finding, + ) + # Attach so tests can inspect. + ctx._captured_findings = findings # type: ignore[attr-defined] + return ctx + + return _make + + +def _make_album_row(*, id_, title='Test Album', artist_id=10, artist_name='Test Artist'): + """Match the row shape ``_load_albums`` returns.""" + return { + 'id': id_, + 'title': title, + 'artist_id': artist_id, + 'artist_name': artist_name, + } + + +def _stub_preview(monkeypatch, response_by_album_id: dict): + """Patch ``preview_album_reorganize`` import inside scan() so it + returns a canned response per album_id.""" + from core import library_reorganize as core_lr + + def _fake_preview(*, album_id, **kwargs): + return response_by_album_id.get(album_id) or { + 'success': False, 'status': 'no_album', 'tracks': [], + } + monkeypatch.setattr(core_lr, 'preview_album_reorganize', _fake_preview) + + +# ── core delegation contract ───────────────────────────────────────── + + +def test_scan_skips_when_file_organization_disabled(make_context): + """Pin: file_organization.enabled=False → scan returns immediately + with empty result, no DB iteration, no preview calls.""" + db = _FakeDB([]) + cm = _FakeConfigManager(file_org_enabled=False) + ctx = make_context(db=db, cm=cm) + + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert result.scanned == 0 + assert result.findings_created == 0 + assert ctx._captured_findings == [] # type: ignore[attr-defined] + + +def test_scan_returns_empty_when_no_albums(make_context): + """Pin: empty DB → empty result, no errors.""" + db = _FakeDB([]) + ctx = make_context(db=db) + + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert result.scanned == 0 + assert result.findings_created == 0 + assert result.errors == 0 + + +def test_scan_emits_path_mismatch_finding_for_each_changed_track(make_context, monkeypatch): + """Pin: dry-run mode emits one finding per matched-but-not-unchanged + track returned by the planner. Album with two tracks, both + mismatched → two findings.""" + db = _FakeDB([_make_album_row(id_='A1')]) + _stub_preview(monkeypatch, { + 'A1': { + 'success': True, 'status': 'planned', + 'source': 'spotify', + 'album': 'Album One', + 'artist': 'Artist One', + 'tracks': [ + { + 'track_id': 't1', 'title': 'Track One', 'track_number': 1, + 'current_path': 'old/path/01 - Track One.flac', + 'new_path': 'new/path/01 - Track One.flac', + 'matched': True, 'unchanged': False, 'file_exists': True, + }, + { + 'track_id': 't2', 'title': 'Track Two', 'track_number': 2, + 'current_path': 'old/path/02 - Track Two.flac', + 'new_path': 'new/path/02 - Track Two.flac', + 'matched': True, 'unchanged': False, 'file_exists': True, + }, + ], + }, + }) + ctx = make_context(db=db, dry_run=True) + + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert result.findings_created == 2 + assert result.scanned == 2 + findings = ctx._captured_findings # type: ignore[attr-defined] + titles = {f['title'] for f in findings} + assert any('Track One' in t for t in titles) + assert any('Track Two' in t for t in titles) + + +def test_scan_skips_unchanged_tracks(make_context, monkeypatch): + """Pin: tracks with unchanged=True don't produce findings — they're + already at the right path.""" + db = _FakeDB([_make_album_row(id_='A1')]) + _stub_preview(monkeypatch, { + 'A1': { + 'success': True, 'status': 'planned', + 'source': 'spotify', + 'album': 'Album One', 'artist': 'Artist One', + 'tracks': [ + { + 'track_id': 't1', 'title': 'Track One', + 'current_path': 'right/path/01 - Track One.flac', + 'new_path': 'right/path/01 - Track One.flac', + 'matched': True, 'unchanged': True, 'file_exists': True, + }, + ], + }, + }) + ctx = make_context(db=db, dry_run=True) + + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert result.findings_created == 0 + assert result.scanned == 1 + + +def test_scan_skips_unmatched_tracks_within_planned_album(make_context, monkeypatch): + """Pin: tracks the planner couldn't match (matched=False) are + skipped — no path was computed for them, can't reorganize.""" + db = _FakeDB([_make_album_row(id_='A1')]) + _stub_preview(monkeypatch, { + 'A1': { + 'success': True, 'status': 'planned', + 'source': 'spotify', + 'album': 'Album One', 'artist': 'Artist One', + 'tracks': [ + { + 'track_id': 't1', 'title': 'Bonus Track', + 'current_path': 'old/path/12 - Bonus.flac', + 'new_path': '', + 'matched': False, 'unchanged': False, 'file_exists': True, + 'reason': 'No matching track in spotify tracklist', + }, + ], + }, + }) + ctx = make_context(db=db, dry_run=True) + + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert result.findings_created == 0 + assert result.scanned == 1 + + +def test_scan_skips_tracks_with_missing_files(make_context, monkeypatch): + """Pin: file_exists=False → not eligible for move (handled by the + Dead File Cleaner job instead). No path_mismatch finding emitted.""" + db = _FakeDB([_make_album_row(id_='A1')]) + _stub_preview(monkeypatch, { + 'A1': { + 'success': True, 'status': 'planned', + 'source': 'spotify', + 'album': 'Album One', 'artist': 'Artist One', + 'tracks': [ + { + 'track_id': 't1', 'title': 'Missing Track', + 'current_path': 'gone/01 - Missing.flac', + 'new_path': 'new/01 - Missing.flac', + 'matched': True, 'unchanged': False, 'file_exists': False, + }, + ], + }, + }) + ctx = make_context(db=db, dry_run=True) + + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert result.findings_created == 0 + + +def test_scan_emits_album_needs_enrichment_when_planner_returns_no_source_id(make_context, monkeypatch): + """Pin: planner returns status='no_source_id' → emit ONE + album-level finding ('needs enrichment') instead of N per-track + 'no source' findings (which would clutter the UI).""" + db = _FakeDB([_make_album_row(id_='A1', title='Unenriched Album')]) + _stub_preview(monkeypatch, { + 'A1': { + 'success': False, 'status': 'no_source_id', + 'source': None, + 'album': 'Unenriched Album', 'artist': 'Some Artist', + 'tracks': [ + {'track_id': 't1', 'title': 'Track 1', 'matched': False, 'reason': '...'}, + {'track_id': 't2', 'title': 'Track 2', 'matched': False, 'reason': '...'}, + ], + }, + }) + ctx = make_context(db=db, dry_run=True) + + job = LibraryReorganizeJob() + result = job.scan(ctx) + + findings = ctx._captured_findings # type: ignore[attr-defined] + assert result.findings_created == 1 + assert findings[0]['finding_type'] == 'album_needs_enrichment' + assert 'Unenriched Album' in findings[0]['title'] + + +def test_scan_skips_albums_planner_reports_as_no_album(make_context, monkeypatch): + """Pin: planner returns 'no_album' (race: album deleted between + SELECT and preview) → silently skipped, no finding, no error.""" + db = _FakeDB([_make_album_row(id_='A1')]) + _stub_preview(monkeypatch, { + 'A1': {'success': False, 'status': 'no_album', 'tracks': []}, + }) + ctx = make_context(db=db, dry_run=True) + + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert result.findings_created == 0 + assert result.errors == 0 + assert result.skipped == 1 + + +def test_scan_handles_preview_exceptions_gracefully(make_context, monkeypatch): + """Pin: preview raising for one album doesn't abort the whole + scan — counts as one error, continues to next album.""" + db = _FakeDB([ + _make_album_row(id_='A1'), + _make_album_row(id_='A2', title='Good Album'), + ]) + from core import library_reorganize as core_lr + + def _flaky(*, album_id, **kwargs): + if album_id == 'A1': + raise RuntimeError("preview boom") + return { + 'success': True, 'status': 'planned', + 'source': 'spotify', + 'album': 'Good Album', 'artist': 'Artist', + 'tracks': [ + { + 'track_id': 't1', 'title': 'Track', + 'current_path': 'old/01 - Track.flac', + 'new_path': 'new/01 - Track.flac', + 'matched': True, 'unchanged': False, 'file_exists': True, + }, + ], + } + monkeypatch.setattr(core_lr, 'preview_album_reorganize', _flaky) + + ctx = make_context(db=db, dry_run=True) + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert result.errors == 1 # A1 failed + assert result.findings_created == 1 # A2 succeeded + + +def test_scan_apply_mode_enqueues_albums_via_reorganize_queue(make_context, monkeypatch): + """Pin: dry_run=False → mismatched albums are bulk-enqueued via + ``core.reorganize_queue.get_queue().enqueue_many(...)``. Repair + job does NOT do file moves itself — delegates to the queue worker + which uses the same code path the per-album modal does.""" + db = _FakeDB([ + _make_album_row(id_='A1', title='First Album', artist_id=10, artist_name='A'), + _make_album_row(id_='A2', title='Second Album', artist_id=20, artist_name='B'), + ]) + _stub_preview(monkeypatch, { + 'A1': { + 'success': True, 'status': 'planned', + 'source': 'spotify', + 'album': 'First Album', 'artist': 'A', + 'tracks': [ + { + 'track_id': 't1', 'title': 'X', + 'current_path': 'old/01 - X.flac', 'new_path': 'new/01 - X.flac', + 'matched': True, 'unchanged': False, 'file_exists': True, + }, + ], + }, + 'A2': { + 'success': True, 'status': 'planned', + 'source': 'deezer', + 'album': 'Second Album', 'artist': 'B', + 'tracks': [ + { + 'track_id': 't2', 'title': 'Y', + 'current_path': 'old/01 - Y.flac', 'new_path': 'new/01 - Y.flac', + 'matched': True, 'unchanged': False, 'file_exists': True, + }, + ], + }, + }) + + enqueue_calls = [] + + class _StubQueue: + def enqueue_many(self, items): + enqueue_calls.append(items) + # Match the real queue's return shape: + # {'enqueued': N, 'already_queued': M, 'total': K} + return {'enqueued': len(items), 'already_queued': 0, 'total': len(items)} + + import core.reorganize_queue as queue_mod + monkeypatch.setattr(queue_mod, 'get_queue', lambda: _StubQueue()) + + ctx = make_context(db=db, dry_run=False) + job = LibraryReorganizeJob() + result = job.scan(ctx) + + assert len(enqueue_calls) == 1 + queued = enqueue_calls[0] + assert {q['album_id'] for q in queued} == {'A1', 'A2'} + assert {q['source'] for q in queued} == {'spotify', 'deezer'} + assert result.auto_fixed == 2 + # Apply mode does NOT emit findings — it enqueues for actual move. + assert result.findings_created == 0 + + +def test_scan_only_iterates_albums_for_active_server(make_context, monkeypatch): + """Pin: multi-server users (Plex + Jellyfin etc) — the job only + iterates albums on the ACTIVE server. Inactive server's rows are + skipped so we don't move files at paths the user can't see in + the artist-detail UI.""" + db = _FakeDB(rows_by_server={ + 'plex': [_make_album_row(id_='plex_album_1', title='Plex-only Album')], + 'jellyfin': [ + _make_album_row(id_='jelly_album_1', title='Jelly Album 1'), + _make_album_row(id_='jelly_album_2', title='Jelly Album 2'), + ], + }) + cm = _FakeConfigManager(active_server='jellyfin') + ctx = make_context(db=db, cm=cm) + + seen_album_ids = [] + + from core import library_reorganize as core_lr + + def _track_calls(*, album_id, **kwargs): + seen_album_ids.append(album_id) + return {'success': False, 'status': 'no_album', 'tracks': []} + monkeypatch.setattr(core_lr, 'preview_album_reorganize', _track_calls) + + job = LibraryReorganizeJob() + job.scan(ctx) + + # Only Jellyfin's two albums were processed; the Plex album was + # filtered out by the SQL active-server clause. + assert sorted(seen_album_ids) == ['jelly_album_1', 'jelly_album_2'] + + +def test_estimate_scope_returns_album_count(monkeypatch): + """Pin: ``estimate_scope`` returns the DB album count (matches + what scan iterates over).""" + cursor = MagicMock() + cursor.fetchone.return_value = (42,) + conn = MagicMock() + conn.cursor.return_value = cursor + + db = MagicMock() + db._get_connection.return_value = conn + + cm = _FakeConfigManager() + ctx = JobContext( + db=db, transfer_folder='/tmp', config_manager=cm, ) - job = lr.LibraryReorganizeJob() - result = job._lookup_years_from_api(SimpleNamespace(report_progress=None, check_stop=lambda: False, sleep_or_stop=lambda *_: False), {('Artist', 'Album')}) - - assert result == {('artist', 'album'): '2022'} - assert deezer_client.calls == [('Artist Album', 3)] - assert spotify_client.calls == [] - - -def test_lookup_years_falls_through_to_later_source(monkeypatch): - deezer_client = _FakeSearchClient('deezer', None) - spotify_client = _FakeSearchClient('spotify', '1999') - - monkeypatch.setattr(lr, 'get_primary_source', lambda: 'deezer') - monkeypatch.setattr(lr, 'get_source_priority', lambda primary: [primary, 'itunes', 'spotify']) - monkeypatch.setattr( - lr, - 'get_client_for_source', - lambda source: {'deezer': deezer_client, 'spotify': spotify_client}.get(source), - ) - - job = lr.LibraryReorganizeJob() - result = job._lookup_years_from_api( - SimpleNamespace(report_progress=None, check_stop=lambda: False, sleep_or_stop=lambda *_: False), - {('Artist', 'Album')}, - ) - - assert result == {('artist', 'album'): '1999'} - assert deezer_client.calls == [('Artist Album', 3)] - assert spotify_client.calls == [('Artist Album', 3)] - - -def test_lookup_years_rechecks_client_availability_per_album(monkeypatch): - availability = {'spotify': True} - - class _SpotifyClient(_FakeSearchClient): - def search_albums(self, query, limit=3): - self.calls.append((query, limit)) - availability['spotify'] = False - return [] - - spotify_client = _SpotifyClient('spotify', None) - itunes_client = _FakeSearchClient('itunes', '2002') - helper_calls = [] - - def fake_get_client_for_source(source): - helper_calls.append(source) - if source == 'spotify' and not availability['spotify']: - return None - return {'spotify': spotify_client, 'itunes': itunes_client}.get(source) - - monkeypatch.setattr(lr, 'get_primary_source', lambda: 'spotify') - monkeypatch.setattr(lr, 'get_source_priority', lambda primary: [primary, 'itunes']) - monkeypatch.setattr(lr, 'get_client_for_source', fake_get_client_for_source) - - job = lr.LibraryReorganizeJob() - result = job._lookup_years_from_api( - SimpleNamespace(report_progress=None, check_stop=lambda: False, sleep_or_stop=lambda *_: False), - {('Artist A', 'Album A'), ('Artist B', 'Album B')}, - ) - - assert result == {('artist a', 'album a'): '2002', ('artist b', 'album b'): '2002'} - assert helper_calls.count('spotify') == 2 - assert helper_calls.count('itunes') == 2 - assert len(spotify_client.calls) == 1 - assert spotify_client.calls[0] in [('Artist A Album A', 3), ('Artist B Album B', 3)] - assert set(itunes_client.calls) == {('Artist A Album A', 3), ('Artist B Album B', 3)} + job = LibraryReorganizeJob() + assert job.estimate_scope(ctx) == 42 diff --git a/tests/test_library_track_identity.py b/tests/test_library_track_identity.py new file mode 100644 index 00000000..b1e24923 --- /dev/null +++ b/tests/test_library_track_identity.py @@ -0,0 +1,412 @@ +"""Tests for the provider-neutral external-ID match helper. + +Discord-reported (CAL): the watchlist scanner re-downloaded a track +already on disk because the library DB had stale album metadata. The +album fuzzy correctly said the names didn't match and the scanner +declared the track missing. The track's stable external IDs (Spotify +ID, Deezer ID, MusicBrainz recording ID, ISRC, etc.) were available on +both sides but never consulted. + +These tests pin the new ID-extraction helper + the library SELECT so +the regression doesn't return. +""" + +from __future__ import annotations + +import sqlite3 +from typing import Any, Dict + +import pytest + +from core.library.track_identity import ( + EXTERNAL_ID_COLUMNS, + extract_external_ids, + find_library_track_by_external_id, +) + + +# --------------------------------------------------------------------------- +# extract_external_ids +# --------------------------------------------------------------------------- + + +class TestExtractExternalIdsFromDirectFields: + def test_spotify_track_with_spotify_id_field(self): + track = {'spotify_id': 'sp1', 'name': 'Hello'} + assert extract_external_ids(track) == {'spotify_id': 'sp1'} + + def test_track_with_alias_spotify_track_id(self): + track = {'spotify_track_id': 'sp1', 'name': 'Hello'} + assert extract_external_ids(track) == {'spotify_id': 'sp1'} + + def test_track_with_uppercase_tag_name(self): + track = {'SPOTIFY_TRACK_ID': 'sp1', 'name': 'Hello'} + assert extract_external_ids(track) == {'spotify_id': 'sp1'} + + def test_itunes_via_trackId_alias(self): + track = {'trackId': 12345, 'name': 'Hello'} + assert extract_external_ids(track) == {'itunes_id': '12345'} + + def test_deezer_via_provider_native_id(self): + track = {'id': 'dz1', 'provider': 'deezer', 'name': 'Hello'} + assert extract_external_ids(track) == {'deezer_id': 'dz1'} + + def test_isrc_extracted(self): + track = {'isrc': 'USRC17607839', 'name': 'Hello'} + assert extract_external_ids(track) == {'isrc': 'USRC17607839'} + + def test_musicbrainz_recording_id_extracted(self): + track = {'musicbrainz_recording_id': 'mb-uuid-1', 'name': 'Hello'} + assert extract_external_ids(track) == {'mbid': 'mb-uuid-1'} + + def test_audiodb_id_via_idTrack_alias(self): + track = {'idTrack': 'adb1', 'name': 'Hello'} + assert extract_external_ids(track) == {'audiodb_id': 'adb1'} + + def test_soul_id_extracted(self): + track = {'soul_id': 'soul-abc', 'name': 'Hello'} + assert extract_external_ids(track) == {'soul_id': 'soul-abc'} + + +class TestExtractExternalIdsFromProviderField: + """The provider field disambiguates a track's native ``id`` field.""" + + def test_provider_spotify_with_native_id(self): + track = {'provider': 'spotify', 'id': 'sp1', 'name': 'Hello'} + assert extract_external_ids(track) == {'spotify_id': 'sp1'} + + def test_provider_itunes_with_native_id(self): + track = {'provider': 'itunes', 'id': 'it1', 'name': 'Hello'} + assert extract_external_ids(track) == {'itunes_id': 'it1'} + + def test_provider_tidal_with_native_id(self): + track = {'provider': 'tidal', 'id': 'td1', 'name': 'Hello'} + assert extract_external_ids(track) == {'tidal_id': 'td1'} + + def test_provider_qobuz_with_native_id(self): + track = {'provider': 'qobuz', 'id': 'qb1', 'name': 'Hello'} + assert extract_external_ids(track) == {'qobuz_id': 'qb1'} + + def test_provider_musicbrainz_with_native_id(self): + track = {'provider': 'musicbrainz', 'id': 'mb-uuid', 'name': 'Hello'} + assert extract_external_ids(track) == {'mbid': 'mb-uuid'} + + def test_provider_hydrabase_with_native_id(self): + track = {'provider': 'hydrabase', 'id': 'hyd1', 'name': 'Hello'} + assert extract_external_ids(track) == {'soul_id': 'hyd1'} + + def test_source_field_treated_same_as_provider(self): + track = {'source': 'deezer', 'id': 'dz1', 'name': 'Hello'} + assert extract_external_ids(track) == {'deezer_id': 'dz1'} + + def test_underscore_source_field_treated_same_as_provider(self): + """Deezer / Discogs / Hydrabase clients tag normalized tracks + with ``_source`` (underscore prefix). Must be recognized as the + provider disambiguator — otherwise watchlist scans against those + sources silently miss the ID match and fall through to fuzzy.""" + track = {'_source': 'deezer', 'id': 'dz1', 'name': 'Hello'} + assert extract_external_ids(track) == {'deezer_id': 'dz1'} + + def test_underscore_source_hydrabase(self): + track = {'_source': 'hydrabase', 'id': 'hyd-soul-1', 'name': 'Hello'} + assert extract_external_ids(track) == {'soul_id': 'hyd-soul-1'} + + def test_native_id_without_provider_is_ignored(self): + """Without a provider field we can't tell which source 'id' belongs to.""" + track = {'id': 'unknown', 'name': 'Hello'} + assert extract_external_ids(track) == {} + + def test_source_hint_disambiguates_when_track_has_no_provider(self): + """Spotify and iTunes raw API responses don't carry a provider / + source / _source field. The watchlist scanner passes the + configured primary source as a hint so the extractor can map + the track's native ``id`` field correctly.""" + track = {'id': 'sp1', 'name': 'Hello'} # No provider field + assert extract_external_ids(track, source_hint='spotify') == {'spotify_id': 'sp1'} + assert extract_external_ids(track, source_hint='itunes') == {'itunes_id': 'sp1'} + assert extract_external_ids(track, source_hint='deezer') == {'deezer_id': 'sp1'} + + def test_source_hint_does_not_override_track_provider(self): + """If the track already carries an explicit provider, that wins + over the hint — defensive against the hint being wrong.""" + track = {'_source': 'deezer', 'id': 'dz1', 'name': 'Hello'} + assert extract_external_ids(track, source_hint='spotify') == {'deezer_id': 'dz1'} + + def test_source_hint_none_is_ignored(self): + track = {'id': 'unknown', 'name': 'Hello'} + assert extract_external_ids(track, source_hint=None) == {} + + +class TestExtractExternalIdsMixedAndDefensive: + def test_track_with_multiple_provider_specific_fields(self): + track = { + 'spotify_id': 'sp1', + 'itunes_id': 'it1', + 'isrc': 'USRC17607839', + 'name': 'Hello', + } + assert extract_external_ids(track) == { + 'spotify_id': 'sp1', + 'itunes_id': 'it1', + 'isrc': 'USRC17607839', + } + + def test_direct_field_takes_precedence_over_provider_native_id(self): + """If both 'spotify_id' and provider/'id' are set, the direct + field wins (already collected first).""" + track = { + 'spotify_id': 'direct-sp', + 'provider': 'spotify', + 'id': 'native-sp', + } + assert extract_external_ids(track) == {'spotify_id': 'direct-sp'} + + def test_object_style_track_supported(self): + class _Track: + def __init__(self): + self.spotify_id = 'sp1' + self.isrc = 'USRC17607839' + self.name = 'Hello' + + assert extract_external_ids(_Track()) == { + 'spotify_id': 'sp1', + 'isrc': 'USRC17607839', + } + + def test_empty_strings_treated_as_missing(self): + track = {'spotify_id': '', 'itunes_id': ' ', 'isrc': None} + assert extract_external_ids(track) == {} + + def test_no_ids_returns_empty_dict(self): + track = {'name': 'Hello', 'duration_ms': 1000} + assert extract_external_ids(track) == {} + + def test_none_track_returns_empty_dict(self): + assert extract_external_ids(None) == {} + + def test_numeric_ids_coerced_to_string(self): + track = {'spotify_id': 12345, 'name': 'Hello'} + assert extract_external_ids(track) == {'spotify_id': '12345'} + + +# --------------------------------------------------------------------------- +# find_library_track_by_external_id +# --------------------------------------------------------------------------- + + +class _FakeDatabase: + """Minimal DB stand-in exposing ``_get_connection()`` like MusicDatabase.""" + + def __init__(self): + self._conn = sqlite3.connect(':memory:') + self._conn.row_factory = sqlite3.Row + # Schema mirrors the columns the helper reads — only the ones we + # actually use need to exist. + self._conn.execute(""" + CREATE TABLE tracks ( + id INTEGER PRIMARY KEY, + title TEXT, + spotify_track_id TEXT, + itunes_track_id TEXT, + deezer_id TEXT, + tidal_id TEXT, + qobuz_id TEXT, + musicbrainz_recording_id TEXT, + audiodb_id TEXT, + soul_id TEXT, + isrc TEXT, + server_source TEXT + ) + """) + self._conn.commit() + + def _get_connection(self): + # Mirror MusicDatabase's pattern: caller closes the returned + # connection. Use a thin wrapper that no-ops close so the in- + # memory DB isn't dropped between calls. + class _NoCloseConn: + def __init__(_self, real): + _self._real = real + + def __getattr__(_self, name): + return getattr(_self._real, name) + + def close(_self): + pass + + return _NoCloseConn(self._conn) + + def insert(self, **kwargs): + cols = ', '.join(kwargs.keys()) + placeholders = ', '.join('?' * len(kwargs)) + self._conn.execute( + f"INSERT INTO tracks ({cols}) VALUES ({placeholders})", + list(kwargs.values()), + ) + self._conn.commit() + + +@pytest.fixture +def db(): + return _FakeDatabase() + + +class TestFindLibraryTrackBySpotifyId: + def test_match_by_spotify_id(self, db): + db.insert(title='Hello', spotify_track_id='sp1') + result = find_library_track_by_external_id(db, external_ids={'spotify_id': 'sp1'}) + assert result is not None + assert result['title'] == 'Hello' + assert result['spotify_track_id'] == 'sp1' + + def test_no_match_returns_none(self, db): + db.insert(title='Hello', spotify_track_id='sp-other') + result = find_library_track_by_external_id(db, external_ids={'spotify_id': 'sp1'}) + assert result is None + + def test_null_column_is_skipped(self, db): + """A library row with NULL spotify_track_id must NOT match an + empty/missing source ID — the IS NOT NULL guard prevents that.""" + db.insert(title='NoIDs') # all IDs NULL + # Empty external_ids → no match + assert find_library_track_by_external_id(db, external_ids={}) is None + + +class TestFindLibraryTrackProviderNeutral: + def test_match_by_itunes_id_when_spotify_missing(self, db): + db.insert(title='Hello', itunes_track_id='it1') + result = find_library_track_by_external_id( + db, external_ids={'itunes_id': 'it1'}, + ) + assert result is not None + assert result['itunes_track_id'] == 'it1' + + def test_match_by_deezer_id(self, db): + db.insert(title='Hello', deezer_id='dz1') + result = find_library_track_by_external_id( + db, external_ids={'deezer_id': 'dz1'}, + ) + assert result is not None + + def test_match_by_tidal_id(self, db): + db.insert(title='Hello', tidal_id='td1') + result = find_library_track_by_external_id( + db, external_ids={'tidal_id': 'td1'}, + ) + assert result is not None + + def test_match_by_qobuz_id(self, db): + db.insert(title='Hello', qobuz_id='qb1') + result = find_library_track_by_external_id( + db, external_ids={'qobuz_id': 'qb1'}, + ) + assert result is not None + + def test_match_by_musicbrainz_recording_id(self, db): + db.insert(title='Hello', musicbrainz_recording_id='mb-uuid') + result = find_library_track_by_external_id( + db, external_ids={'mbid': 'mb-uuid'}, + ) + assert result is not None + + def test_match_by_isrc_across_providers(self, db): + """ISRC is the cross-source identity — a library track imported + from Deezer can be matched against a Spotify scan if both carry + the same ISRC.""" + db.insert(title='Hello', deezer_id='dz1', isrc='USRC17607839') + # Source track has Spotify ID + ISRC; library only has Deezer + ISRC. + # The ISRC bridges them. + result = find_library_track_by_external_id( + db, external_ids={'spotify_id': 'sp-different', 'isrc': 'USRC17607839'}, + ) + assert result is not None + assert result['isrc'] == 'USRC17607839' + + def test_match_by_soul_id(self, db): + db.insert(title='Hello', soul_id='hyd-soul-1') + result = find_library_track_by_external_id( + db, external_ids={'soul_id': 'hyd-soul-1'}, + ) + assert result is not None + + +class TestFindLibraryTrackOrSemantics: + def test_any_one_matching_id_is_enough(self, db): + db.insert(title='Hello', spotify_track_id='sp1') + result = find_library_track_by_external_id( + db, + external_ids={ + 'spotify_id': 'sp1', + 'itunes_id': 'wrong', + 'deezer_id': 'wrong', + }, + ) + assert result is not None + + def test_no_matching_id_returns_none(self, db): + db.insert(title='Hello', spotify_track_id='sp1', itunes_track_id='it1') + result = find_library_track_by_external_id( + db, + external_ids={'deezer_id': 'dz-other'}, + ) + assert result is None + + def test_empty_external_ids_returns_none(self, db): + db.insert(title='Hello', spotify_track_id='sp1') + assert find_library_track_by_external_id(db, external_ids={}) is None + + +class TestFindLibraryTrackServerSourceFilter: + def test_server_source_match(self, db): + db.insert(title='Hello', spotify_track_id='sp1', server_source='plex') + result = find_library_track_by_external_id( + db, external_ids={'spotify_id': 'sp1'}, server_source='plex', + ) + assert result is not None + + def test_server_source_mismatch_with_filter(self, db): + db.insert(title='Hello', spotify_track_id='sp1', server_source='jellyfin') + result = find_library_track_by_external_id( + db, external_ids={'spotify_id': 'sp1'}, server_source='plex', + ) + # Filter excludes jellyfin, so no match. + assert result is None + + def test_null_server_source_passes_filter(self, db): + """Older library rows may have NULL server_source — those should + still match when a filter is applied (defensive).""" + db.insert(title='Hello', spotify_track_id='sp1', server_source=None) + result = find_library_track_by_external_id( + db, external_ids={'spotify_id': 'sp1'}, server_source='plex', + ) + assert result is not None + + def test_no_filter_matches_any_server_source(self, db): + db.insert(title='Hello', spotify_track_id='sp1', server_source='jellyfin') + result = find_library_track_by_external_id( + db, external_ids={'spotify_id': 'sp1'}, server_source=None, + ) + assert result is not None + + +# --------------------------------------------------------------------------- +# EXTERNAL_ID_COLUMNS map sanity +# --------------------------------------------------------------------------- + + +class TestExternalIdColumnsMap: + def test_every_known_id_name_has_a_column(self): + """If extract_external_ids ever adds a new ID name, the map needs + a column entry too — otherwise find_library_track_by_external_id + silently ignores it.""" + # Sample of ID names extract_external_ids can return; keep in sync. + known_id_names = { + 'spotify_id', 'itunes_id', 'deezer_id', 'tidal_id', 'qobuz_id', + 'mbid', 'audiodb_id', 'soul_id', 'isrc', + } + assert set(EXTERNAL_ID_COLUMNS.keys()) == known_id_names + + def test_column_names_are_unique(self): + cols = list(EXTERNAL_ID_COLUMNS.values()) + assert len(cols) == len(set(cols)), \ + f"Duplicate column targets in EXTERNAL_ID_COLUMNS: {cols}" diff --git a/tests/test_lidarr_download_client.py b/tests/test_lidarr_download_client.py new file mode 100644 index 00000000..4469475a --- /dev/null +++ b/tests/test_lidarr_download_client.py @@ -0,0 +1,297 @@ +"""Regression tests for ``core/lidarr_download_client.py``. + +The original implementation had three bugs that prevented Lidarr from +being a viable download source: + +1. **File misfiling.** Lidarr grabs whole albums; the user requested a + specific track. Old code copied every track in the album and reported + ``imported_files[0]`` as ``file_path`` — almost always pointing to + track 1, not the user's actual track. Post-processing then tagged + track 1 with the wrong metadata. +2. **Hardcoded ``metadataProfileId=1``.** On Lidarr installs where the + user deleted/recreated metadata profiles, that id no longer exists + and the artist-add API call fails with HTTP 400. +3. **Polling never broke the outer loop on completion.** The inner + ``break`` only exited the queue iteration, so the outer poll loop + kept spinning until the 600-poll timeout even after the album was + imported. + +These tests pin the fixed behavior in isolation: pure-function helpers +(title similarity, title extraction, normalization) plus integration +tests of the file-picker that go through ``_api_get`` mocked at the +client boundary so we don't need a live Lidarr instance. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from core.lidarr_download_client import LidarrDownloadClient + + +# --------------------------------------------------------------------------- +# Pure-function helpers (no mocking needed) +# --------------------------------------------------------------------------- + + +def test_extract_wanted_track_title_parses_three_part_display() -> None: + title = LidarrDownloadClient._extract_wanted_track_title( +'Kendrick Lamar - GNX - wacced out murals', + ) + assert title == 'wacced out murals' + + +def test_extract_wanted_track_title_handles_dashes_in_track_name() -> None: + """Track titles can contain ' - ' themselves (live versions, mixes, + etc). Rejoin parts[2:] so 'Artist - Album - Track - Live Version' + returns 'Track - Live Version' as the wanted title.""" + title = LidarrDownloadClient._extract_wanted_track_title( + 'Artist - Album - Some Track - Live Version', + ) + assert title == 'Some Track - Live Version' + + +def test_extract_wanted_track_title_returns_empty_for_album_dispatch() -> None: + """Album-level dispatch (no track in display) → empty string, + caller falls back to first-file behavior.""" + title = LidarrDownloadClient._extract_wanted_track_title( +'Kendrick Lamar - GNX', + ) + assert title == '' + + +def test_extract_wanted_track_title_handles_empty_input() -> None: + title = LidarrDownloadClient._extract_wanted_track_title('') + assert title == '' + + +def test_normalize_for_match_strips_punctuation_and_lowercases() -> None: + norm = LidarrDownloadClient._normalize_for_match + assert norm("M.A.A.D City") == 'maad city' + assert norm("Don't Kill My Vibe") == 'dont kill my vibe' + assert norm("HUMBLE.") == 'humble' + assert norm(" Multiple Spaces ") == 'multiple spaces' + assert norm('') == '' + + +def test_title_similarity_exact_match() -> None: + sim = LidarrDownloadClient._title_similarity + assert sim('humble', 'humble') == 1.0 + + +def test_title_similarity_substring_match() -> None: + sim = LidarrDownloadClient._title_similarity + # 'mine' fully contained in 'mine taylors version' + assert sim('mine', 'mine taylors version') == 0.85 + # Reverse direction + assert sim('mine taylors version', 'mine') == 0.85 + + +def test_title_similarity_token_overlap() -> None: + """Token overlap ratio for partial matches: shared tokens / union.""" + sim = LidarrDownloadClient._title_similarity + # 2 shared tokens, 3 total → 2/3 ≈ 0.67 + score = sim('hello world', 'hello cruel world') + assert 0.6 < score < 0.7 + + +def test_title_similarity_no_overlap() -> None: + sim = LidarrDownloadClient._title_similarity + assert sim('completely different', 'unrelated track') == 0.0 + + +def test_title_similarity_empty_inputs() -> None: + sim = LidarrDownloadClient._title_similarity + assert sim('', 'something') == 0.0 + assert sim('something', '') == 0.0 + assert sim('', '') == 0.0 + + +# --------------------------------------------------------------------------- +# File-picker (mocks Lidarr API responses) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def client(tmp_path: Path) -> LidarrDownloadClient: + """Construct a client without touching real Lidarr config.""" + c = LidarrDownloadClient(download_path=str(tmp_path / 'downloads')) + return c + + +def test_pick_track_file_for_wanted_returns_matching_path(client, tmp_path: Path) -> None: + """Happy path: tracks list includes the wanted title, trackfile API + returns a real on-disk path. Picker returns it.""" + real_file = tmp_path / 'wacced.flac' + real_file.write_bytes(b'audio') + + tracks_response = [ + {'title': 'wacced out murals', 'trackFileId': 42}, + {'title': 'squabble up', 'trackFileId': 43}, + ] + trackfile_response = {'id': 42, 'path': str(real_file)} + + def _api_get_stub(endpoint, params=None): + if endpoint == 'track': + return tracks_response + if endpoint == 'trackfile/42': + return trackfile_response + return None + + with patch.object(client, '_api_get', side_effect=_api_get_stub): + result = client._pick_track_file_for_wanted( + lidarr_album_id=999, wanted_title='wacced out murals', + ) + assert result == str(real_file) + + +def test_pick_track_file_for_wanted_handles_punctuation_difference(client, tmp_path: Path) -> None: + """Lidarr says 'm.A.A.d city', user dispatched as 'maad city'. + Normalization should match them via token-equality after stripping + punctuation.""" + real_file = tmp_path / 'maad.flac' + real_file.write_bytes(b'audio') + + tracks_response = [ + {'title': 'm.A.A.d city', 'trackFileId': 7}, + ] + + def _api_get_stub(endpoint, params=None): + if endpoint == 'track': + return tracks_response + if endpoint == 'trackfile/7': + return {'id': 7, 'path': str(real_file)} + return None + + with patch.object(client, '_api_get', side_effect=_api_get_stub): + result = client._pick_track_file_for_wanted( + lidarr_album_id=999, wanted_title='maad city', + ) + assert result == str(real_file) + + +def test_pick_track_file_for_wanted_returns_none_below_threshold(client) -> None: + """If no track in the album is similar enough to the wanted title, + return None so caller falls back to first-imported behavior.""" + tracks_response = [ + {'title': 'completely unrelated song', 'trackFileId': 1}, + {'title': 'another unrelated', 'trackFileId': 2}, + ] + with patch.object(client, '_api_get', return_value=tracks_response): + result = client._pick_track_file_for_wanted( + lidarr_album_id=999, wanted_title='wacced out murals', + ) + assert result is None + + +def test_pick_track_file_for_wanted_returns_none_for_empty_wanted(client) -> None: + """Empty wanted_title → return None (album-level dispatch path).""" + with patch.object(client, '_api_get') as mock_api: + result = client._pick_track_file_for_wanted( + lidarr_album_id=999, wanted_title='', + ) + assert result is None + # API never called — saves a roundtrip when we know we don't need it + mock_api.assert_not_called() + + +def test_pick_track_file_for_wanted_returns_none_when_track_api_fails(client) -> None: + """Defensive: if Lidarr's track API returns None or non-list, + don't crash — return None and let caller fall back.""" + with patch.object(client, '_api_get', return_value=None): + result = client._pick_track_file_for_wanted( + lidarr_album_id=999, wanted_title='anything', + ) + assert result is None + + +def test_pick_track_file_for_wanted_skips_tracks_without_trackfileid(client, tmp_path: Path) -> None: + """Tracks not yet downloaded (no trackFileId) must be skipped — only + consider tracks that actually have an imported file.""" + real_file = tmp_path / 'real.flac' + real_file.write_bytes(b'audio') + + tracks_response = [ + {'title': 'wacced out murals', 'trackFileId': None}, # not imported + {'title': 'wacced out murals (alt)', 'trackFileId': 99}, + ] + + def _api_get_stub(endpoint, params=None): + if endpoint == 'track': + return tracks_response + if endpoint == 'trackfile/99': + return {'id': 99, 'path': str(real_file)} + return None + + with patch.object(client, '_api_get', side_effect=_api_get_stub): + result = client._pick_track_file_for_wanted( + lidarr_album_id=999, wanted_title='wacced out murals', + ) + # Picked the second track because the first has no trackFileId + assert result == str(real_file) + + +def test_pick_track_file_for_wanted_returns_none_when_file_missing_on_disk(client, tmp_path: Path) -> None: + """Lidarr might claim a path exists but the file was moved/deleted + between Lidarr's import and our copy. Return None defensively so + caller falls back.""" + tracks_response = [{'title': 'humble', 'trackFileId': 1}] + trackfile_response = {'id': 1, 'path': str(tmp_path / 'does_not_exist.flac')} + + def _api_get_stub(endpoint, params=None): + if endpoint == 'track': + return tracks_response + if endpoint == 'trackfile/1': + return trackfile_response + return None + + with patch.object(client, '_api_get', side_effect=_api_get_stub): + result = client._pick_track_file_for_wanted( + lidarr_album_id=999, wanted_title='humble', + ) + assert result is None + + +# --------------------------------------------------------------------------- +# Metadata profile id resolution +# --------------------------------------------------------------------------- + + +def test_get_metadata_profile_id_returns_first_available(client) -> None: + """When Lidarr returns a list, pick the first id (matches the + behavior of `_get_quality_profile_id`).""" + profiles = [ + {'id': 5, 'name': 'Standard'}, + {'id': 7, 'name': 'Custom'}, + ] + with patch.object(client, '_api_get', return_value=profiles): + result = client._get_metadata_profile_id() + assert result == 5 + + +def test_get_metadata_profile_id_falls_back_to_one_on_api_failure(client) -> None: + """If the API call returns None (network error / endpoint missing), + return 1 — preserves the previous hardcode as a safety net.""" + with patch.object(client, '_api_get', return_value=None): + result = client._get_metadata_profile_id() + assert result == 1 + + +def test_get_metadata_profile_id_falls_back_when_no_id_field(client) -> None: + """Defensive against malformed responses (profile dicts without id).""" + with patch.object(client, '_api_get', + return_value=[{'name': 'No Id Field'}]): + result = client._get_metadata_profile_id() + assert result == 1 + + +def test_get_metadata_profile_id_falls_back_for_non_list(client) -> None: + """Lidarr API quirk: some endpoints return dicts instead of lists. + Don't crash — fall back to 1.""" + with patch.object(client, '_api_get', return_value={'totalCount': 0}): + result = client._get_metadata_profile_id() + assert result == 1 diff --git a/tests/test_missing_cover_art.py b/tests/test_missing_cover_art.py index cb89896c..5313633f 100644 --- a/tests/test_missing_cover_art.py +++ b/tests/test_missing_cover_art.py @@ -119,7 +119,8 @@ def _make_context(conn, prefer_source=None): wait_if_paused=lambda: False, update_progress=lambda *args, **kwargs: None, report_progress=lambda *args, **kwargs: None, - create_finding=lambda **kwargs: findings.append(kwargs), + # Mirror real `_create_finding` contract: True on insert. + create_finding=lambda **kwargs: (findings.append(kwargs) or True), findings=findings, ) diff --git a/tests/test_provenance_id_persistence.py b/tests/test_provenance_id_persistence.py new file mode 100644 index 00000000..19b24b2a --- /dev/null +++ b/tests/test_provenance_id_persistence.py @@ -0,0 +1,358 @@ +"""Regression tests for the post-processing → provenance → tracks ID flow. + +Companion to test_library_track_identity.py. The watchlist external-ID +match (PR #470) closed the demand side: when the watchlist asks "do we +have this track?", it queries by Spotify/iTunes/Deezer/etc. IDs before +falling back to fuzzy. But for users on Plex / Jellyfin / Navidrome, +the ``tracks.spotify_track_id`` column only gets populated by +asynchronous enrichment workers — sometimes hours after the file is +written. During that window the ID match falls through to fuzzy and +the bug returns. + +This PR closes the supply side: the IDs we already collect at +post-processing time get persisted to ``track_downloads``, and the +media-server sync code copies them onto the new ``tracks`` row +immediately. These tests pin: + +1. Schema migration adds the new ID columns + indexes +2. ``record_track_download`` accepts and persists the new kwargs +3. ``get_provenance_by_file_path`` finds rows by exact + suffix match +4. ``backfill_track_external_ids_from_provenance`` copies IDs onto a + tracks row idempotently (COALESCE — preserves existing values) +5. ``find_provenance_by_external_id`` queries the new columns +""" + +from __future__ import annotations + +import os +import sqlite3 +import tempfile +from pathlib import Path +from typing import Any, Dict + +import pytest + + +@pytest.fixture +def db_path(tmp_path: Path): + return tmp_path / "test_music.db" + + +@pytest.fixture +def db(db_path: Path, monkeypatch): + """Real MusicDatabase against a tmp SQLite file so the schema + migration runs end-to-end (validates the ALTER TABLE additions).""" + monkeypatch.setenv('DATABASE_PATH', str(db_path)) + # MusicDatabase is heavy; isolate to a fresh import each test so + # other tests don't get our env-var pollution. + import importlib + import database.music_database as music_db_module + importlib.reload(music_db_module) + db = music_db_module.MusicDatabase(str(db_path)) + yield db + + +# --------------------------------------------------------------------------- +# Schema migration +# --------------------------------------------------------------------------- + + +class TestSchemaMigration: + def test_track_downloads_has_new_external_id_columns(self, db): + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute("PRAGMA table_info(track_downloads)") + cols = {row[1] for row in cursor.fetchall()} + + assert 'spotify_track_id' in cols + assert 'itunes_track_id' in cols + assert 'deezer_track_id' in cols + assert 'tidal_track_id' in cols + assert 'qobuz_track_id' in cols + assert 'musicbrainz_recording_id' in cols + assert 'audiodb_id' in cols + assert 'soul_id' in cols + assert 'isrc' in cols + + def test_track_downloads_has_external_id_indexes(self, db): + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute( + "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='track_downloads'" + ) + idx_names = {row[0] for row in cursor.fetchall()} + + assert 'idx_td_spotify_id' in idx_names + assert 'idx_td_itunes_id' in idx_names + assert 'idx_td_deezer_id' in idx_names + assert 'idx_td_isrc' in idx_names + + +# --------------------------------------------------------------------------- +# record_track_download persists IDs +# --------------------------------------------------------------------------- + + +class TestRecordTrackDownloadPersistsIds: + def test_persists_all_external_ids(self, db): + rec_id = db.record_track_download( + file_path='/lib/Artist/Album/Track.mp3', + source_service='soulseek', + source_username='user1', + source_filename='Track.mp3', + track_title='Track', + spotify_track_id='sp1', + itunes_track_id='it1', + deezer_track_id='dz1', + tidal_track_id='td1', + qobuz_track_id='qb1', + musicbrainz_recording_id='mb-uuid-1', + audiodb_id='adb1', + soul_id='hyd-soul-1', + isrc='USRC17607839', + ) + assert rec_id is not None + + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute( + "SELECT spotify_track_id, itunes_track_id, deezer_track_id, " + "tidal_track_id, qobuz_track_id, musicbrainz_recording_id, " + "audiodb_id, soul_id, isrc FROM track_downloads WHERE id = ?", + (rec_id,), + ) + row = tuple(cursor.fetchone()) + assert row == ( + 'sp1', 'it1', 'dz1', 'td1', 'qb1', 'mb-uuid-1', + 'adb1', 'hyd-soul-1', 'USRC17607839', + ) + + def test_omitted_ids_persist_as_null(self, db): + """Backward compat — callers that don't pass the new kwargs + still work, columns just stay NULL.""" + rec_id = db.record_track_download( + file_path='/lib/Artist/Album/Track.mp3', + source_service='soulseek', + source_username='user1', + source_filename='Track.mp3', + track_title='Track', + ) + assert rec_id is not None + + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute("SELECT spotify_track_id FROM track_downloads WHERE id = ?", (rec_id,)) + assert cursor.fetchone()[0] is None + + +# --------------------------------------------------------------------------- +# get_provenance_by_file_path +# --------------------------------------------------------------------------- + + +class TestGetProvenanceByFilePath: + def test_exact_match(self, db): + db.record_track_download( + file_path='/lib/Artist/Album/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + spotify_track_id='sp1', + ) + result = db.get_provenance_by_file_path('/lib/Artist/Album/Track.mp3') + assert result is not None + assert result['spotify_track_id'] == 'sp1' + + def test_returns_none_when_no_match(self, db): + result = db.get_provenance_by_file_path('/nonexistent/path.mp3') + assert result is None + + def test_returns_none_for_empty_path(self, db): + assert db.get_provenance_by_file_path('') is None + assert db.get_provenance_by_file_path(None) is None + + def test_basename_suffix_fallback(self, db): + """Recorded path differs from queried path by mount root — + common when SoulSync container writes under /app/Transfer + but Plex container reports the same file as /media/Music.""" + db.record_track_download( + file_path='/app/Transfer/Artist/Album/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + spotify_track_id='sp1', + ) + result = db.get_provenance_by_file_path('/media/Music/Artist/Album/Track.mp3') + assert result is not None + assert result['spotify_track_id'] == 'sp1' + + def test_returns_most_recent_when_multiple(self, db): + """Same file_path can have multiple download records (re-downloads, + retries). Most recent wins.""" + db.record_track_download( + file_path='/lib/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + spotify_track_id='sp-old', + ) + db.record_track_download( + file_path='/lib/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + spotify_track_id='sp-new', + ) + result = db.get_provenance_by_file_path('/lib/Track.mp3') + assert result['spotify_track_id'] == 'sp-new' + + +# --------------------------------------------------------------------------- +# backfill_track_external_ids_from_provenance +# --------------------------------------------------------------------------- + + +class TestBackfillTrackExternalIdsFromProvenance: + def _seed_artist_album_and_track(self, db, *, track_id, file_path): + """Insert a minimal artists/albums/tracks chain so backfill has + a row to update.""" + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute( + "INSERT INTO artists (id, name, server_source) VALUES (?, ?, 'plex')", + ('artist-1', 'Test Artist'), + ) + cursor.execute( + "INSERT INTO albums (id, artist_id, title, server_source) VALUES (?, ?, ?, 'plex')", + ('album-1', 'artist-1', 'Test Album'), + ) + cursor.execute( + "INSERT INTO tracks (id, album_id, artist_id, title, file_path, server_source) " + "VALUES (?, ?, ?, ?, ?, 'plex')", + (track_id, 'album-1', 'artist-1', 'Test Track', file_path), + ) + conn.commit() + + def test_copies_all_ids_when_tracks_columns_empty(self, db): + self._seed_artist_album_and_track(db, track_id='t1', file_path='/lib/Track.mp3') + db.record_track_download( + file_path='/lib/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + spotify_track_id='sp1', + deezer_track_id='dz1', + isrc='USRC17607839', + ) + + updated = db.backfill_track_external_ids_from_provenance('t1', '/lib/Track.mp3') + assert updated > 0 + + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute( + "SELECT spotify_track_id, deezer_id, isrc FROM tracks WHERE id = ?", + ('t1',), + ) + assert tuple(cursor.fetchone()) == ('sp1', 'dz1', 'USRC17607839') + + def test_preserves_existing_ids(self, db): + """COALESCE-update — if the enrichment worker already wrote a + spotify_track_id, the provenance backfill must NOT overwrite it + (enrichment is generally more authoritative for late binding).""" + self._seed_artist_album_and_track(db, track_id='t1', file_path='/lib/Track.mp3') + + # Pre-populate spotify_track_id with the enrichment-worker value + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute("UPDATE tracks SET spotify_track_id = 'sp-from-enrichment' WHERE id = ?", ('t1',)) + conn.commit() + + # Provenance has a different value + db.record_track_download( + file_path='/lib/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + spotify_track_id='sp-from-provenance', + deezer_track_id='dz1', # This one IS missing on tracks, should backfill + ) + + db.backfill_track_external_ids_from_provenance('t1', '/lib/Track.mp3') + + cursor.execute("SELECT spotify_track_id, deezer_id FROM tracks WHERE id = ?", ('t1',)) + row = cursor.fetchone() + assert row[0] == 'sp-from-enrichment', "Existing spotify_track_id must be preserved" + assert row[1] == 'dz1', "Empty deezer_id should be filled from provenance" + + def test_returns_zero_when_no_provenance(self, db): + self._seed_artist_album_and_track(db, track_id='t1', file_path='/lib/Track.mp3') + # No record_track_download call — no provenance row exists + updated = db.backfill_track_external_ids_from_provenance('t1', '/lib/Track.mp3') + assert updated == 0 + + def test_returns_zero_for_empty_inputs(self, db): + assert db.backfill_track_external_ids_from_provenance(None, '/lib/Track.mp3') == 0 + assert db.backfill_track_external_ids_from_provenance('t1', None) == 0 + assert db.backfill_track_external_ids_from_provenance('t1', '') == 0 + + +# --------------------------------------------------------------------------- +# find_provenance_by_external_id +# --------------------------------------------------------------------------- + + +class TestFindProvenanceByExternalId: + def test_match_by_spotify_id(self, db): + db.record_track_download( + file_path='/lib/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + spotify_track_id='sp1', + ) + + from core.library.track_identity import find_provenance_by_external_id + result = find_provenance_by_external_id(db, external_ids={'spotify_id': 'sp1'}) + assert result is not None + assert result['file_path'] == '/lib/Track.mp3' + assert result['spotify_track_id'] == 'sp1' + + def test_match_by_isrc(self, db): + db.record_track_download( + file_path='/lib/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + isrc='USRC17607839', + ) + + from core.library.track_identity import find_provenance_by_external_id + result = find_provenance_by_external_id(db, external_ids={'isrc': 'USRC17607839'}) + assert result is not None + + def test_returns_none_when_no_match(self, db): + from core.library.track_identity import find_provenance_by_external_id + result = find_provenance_by_external_id(db, external_ids={'spotify_id': 'sp-other'}) + assert result is None + + def test_returns_none_for_empty_external_ids(self, db): + from core.library.track_identity import find_provenance_by_external_id + assert find_provenance_by_external_id(db, external_ids={}) is None + + def test_returns_most_recent_when_multiple_matches(self, db): + """Re-downloads create multiple rows. Newest wins.""" + db.record_track_download( + file_path='/lib/Track-v1.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + spotify_track_id='sp1', + ) + db.record_track_download( + file_path='/lib/Track-v2.mp3', + source_service='tidal', source_username='tidal', source_filename='Track.flac', + spotify_track_id='sp1', + ) + + from core.library.track_identity import find_provenance_by_external_id + result = find_provenance_by_external_id(db, external_ids={'spotify_id': 'sp1'}) + assert result['file_path'] == '/lib/Track-v2.mp3' + + def test_or_semantics_across_id_types(self, db): + """Provenance has only ISRC; source asks with multiple IDs incl. ISRC. + Match should fire on ISRC.""" + db.record_track_download( + file_path='/lib/Track.mp3', + source_service='soulseek', source_username='u', source_filename='Track.mp3', + isrc='USRC17607839', + ) + + from core.library.track_identity import find_provenance_by_external_id + result = find_provenance_by_external_id(db, external_ids={ + 'spotify_id': 'sp-mismatch', + 'isrc': 'USRC17607839', + }) + assert result is not None diff --git a/tests/test_qobuz_credential_sync.py b/tests/test_qobuz_credential_sync.py new file mode 100644 index 00000000..4432b28e --- /dev/null +++ b/tests/test_qobuz_credential_sync.py @@ -0,0 +1,289 @@ +"""Regression tests for QobuzClient.reload_credentials. + +Discord-reported (Foxxify): logging in via the Qobuz Connect button on +Settings showed "Connected: (Active)" but underneath an error +"Qobuz not authenticated...", and the dashboard indicator stayed +yellow even after a successful login. + +Root cause: SoulSync runs two QobuzClient instances side by side — one +through ``download_orchestrator.client('qobuz')`` for the auth-flow endpoints, and a +second owned by the enrichment worker thread for thread safety. Login +only updated the first instance's in-memory state. The dashboard's +"configured" check (and the connection-test step) read the worker +instance, which still believed itself unauthenticated until the next +process restart. + +The fix adds ``QobuzClient.reload_credentials()`` — a public, +network-free method that re-reads the saved session from config and +updates the instance's in-memory state + session headers. Called from +the auth login / token / logout endpoints to keep the worker instance +in lockstep with the auth instance. +""" + +from __future__ import annotations + +import sys +import types +from typing import Any, Dict + +import pytest + + +# --------------------------------------------------------------------------- +# Stubs for heavyweight dependencies that QobuzClient pulls in at import time +# --------------------------------------------------------------------------- + + +@pytest.fixture +def qobuz_client_module(): + """Import core.qobuz_client with config_manager stubbed to a mutable + in-memory dict so we can drive `qobuz.session` from the test. + + Snapshots and restores sys.modules entries on teardown — without + this, every downstream test that imports config.settings would + receive our stub and the real config_manager.get would no longer + reach the live config (which breaks tests like + test_tidal_auth_instructions that monkeypatch config_manager.get + directly). + """ + config_state: Dict[str, Any] = {} + + class _StubConfigManager: + def get(self, key, default=None): + cur: Any = config_state + for part in key.split('.'): + if isinstance(cur, dict) and part in cur: + cur = cur[part] + else: + return default + return cur + + def set(self, key, value): + cur: Any = config_state + parts = key.split('.') + for part in parts[:-1]: + cur = cur.setdefault(part, {}) + cur[parts[-1]] = value + + # Snapshot what we are about to mutate so teardown can put it back. + original_modules = { + name: sys.modules.get(name) + for name in ('config', 'config.settings', 'core.qobuz_client') + } + + if 'config' not in sys.modules: + sys.modules['config'] = types.ModuleType('config') + settings_mod = types.ModuleType('config.settings') + settings_mod.config_manager = _StubConfigManager() + sys.modules['config.settings'] = settings_mod + + sys.modules.pop('core.qobuz_client', None) + try: + import core.qobuz_client as qobuz_client_module + yield qobuz_client_module, config_state + finally: + # Restore each entry — set back to original, or pop if it didn't + # exist beforehand. This protects every downstream test that + # imports any of these modules. + for name, original in original_modules.items(): + if original is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = original + + +@pytest.fixture +def fresh_client(qobuz_client_module): + """A QobuzClient instance with no saved session — the constructor's + ``_restore_session()`` call is a no-op when config is empty.""" + module, _config = qobuz_client_module + return module.QobuzClient() + + +# --------------------------------------------------------------------------- +# reload_credentials — populates an empty client from saved config +# --------------------------------------------------------------------------- + + +class TestReloadCredentialsPopulatesFromConfig: + def test_picks_up_token_app_id_and_app_secret(self, qobuz_client_module, fresh_client): + _module, config = qobuz_client_module + config['qobuz'] = { + 'session': { + 'app_id': 'APP-1', + 'app_secret': 'SECRET-1', + 'user_auth_token': 'TOKEN-1', + } + } + # Initial state — empty (constructor saw empty config). + assert fresh_client.user_auth_token is None + assert fresh_client.app_id is None + assert fresh_client.app_secret is None + + fresh_client.reload_credentials() + + assert fresh_client.user_auth_token == 'TOKEN-1' + assert fresh_client.app_id == 'APP-1' + assert fresh_client.app_secret == 'SECRET-1' + + def test_session_headers_get_set(self, qobuz_client_module, fresh_client): + _module, config = qobuz_client_module + config['qobuz'] = { + 'session': { + 'app_id': 'APP-1', + 'app_secret': 'SECRET-1', + 'user_auth_token': 'TOKEN-1', + } + } + fresh_client.reload_credentials() + assert fresh_client.session.headers.get('X-App-Id') == 'APP-1' + assert fresh_client.session.headers.get('X-User-Auth-Token') == 'TOKEN-1' + + def test_authenticated_after_reload(self, qobuz_client_module, fresh_client): + _module, config = qobuz_client_module + config['qobuz'] = { + 'session': { + 'app_id': 'APP-1', + 'app_secret': 'SECRET-1', + 'user_auth_token': 'TOKEN-1', + } + } + fresh_client.reload_credentials() + assert fresh_client.is_authenticated() is True + + +# --------------------------------------------------------------------------- +# reload_credentials — clears state when config is wiped (logout path) +# --------------------------------------------------------------------------- + + +class TestReloadCredentialsClearsOnEmptyConfig: + def test_clears_token_app_id_and_app_secret(self, qobuz_client_module, fresh_client): + _module, config = qobuz_client_module + # Pre-populate + config['qobuz'] = { + 'session': { + 'app_id': 'APP-1', + 'app_secret': 'SECRET-1', + 'user_auth_token': 'TOKEN-1', + } + } + fresh_client.reload_credentials() + assert fresh_client.user_auth_token == 'TOKEN-1' + + # Simulate logout — config wiped + config['qobuz']['session'] = {} + fresh_client.reload_credentials() + + assert fresh_client.user_auth_token is None + assert fresh_client.app_id is None + assert fresh_client.app_secret is None + + def test_session_headers_get_cleared(self, qobuz_client_module, fresh_client): + _module, config = qobuz_client_module + config['qobuz'] = { + 'session': { + 'app_id': 'APP-1', + 'app_secret': 'SECRET-1', + 'user_auth_token': 'TOKEN-1', + } + } + fresh_client.reload_credentials() + assert 'X-User-Auth-Token' in fresh_client.session.headers + + config['qobuz']['session'] = {} + fresh_client.reload_credentials() + assert 'X-User-Auth-Token' not in fresh_client.session.headers + assert 'X-App-Id' not in fresh_client.session.headers + + def test_user_info_reset_when_token_cleared(self, qobuz_client_module, fresh_client): + """When the token gets cleared, stale user_info should not survive + — otherwise downstream code could think a user is still attached + to an unauthenticated instance.""" + _module, config = qobuz_client_module + config['qobuz'] = { + 'session': { + 'app_id': 'APP-1', + 'app_secret': 'SECRET-1', + 'user_auth_token': 'TOKEN-1', + } + } + fresh_client.reload_credentials() + fresh_client.user_info = {'display_name': 'someone'} + + config['qobuz']['session'] = {} + fresh_client.reload_credentials() + assert fresh_client.user_info is None + + def test_not_authenticated_after_clear(self, qobuz_client_module, fresh_client): + _module, config = qobuz_client_module + config['qobuz'] = { + 'session': { + 'app_id': 'APP-1', + 'app_secret': 'SECRET-1', + 'user_auth_token': 'TOKEN-1', + } + } + fresh_client.reload_credentials() + config['qobuz']['session'] = {} + fresh_client.reload_credentials() + assert fresh_client.is_authenticated() is False + + +# --------------------------------------------------------------------------- +# reload_credentials — defensive against missing config keys +# --------------------------------------------------------------------------- + + +class TestReloadCredentialsDefensive: + def test_no_qobuz_key_at_all_clears_state(self, qobuz_client_module, fresh_client): + _module, _config = qobuz_client_module + # Set then tear down so there's nothing in config + fresh_client.app_id = 'X' + fresh_client.user_auth_token = 'Y' + fresh_client.reload_credentials() + assert fresh_client.app_id is None + assert fresh_client.user_auth_token is None + + def test_partial_session_doesnt_crash(self, qobuz_client_module, fresh_client): + """If only token is in config but app_id/secret missing, no crash — + client just isn't authenticated.""" + _module, config = qobuz_client_module + config['qobuz'] = {'session': {'user_auth_token': 'TOKEN-1'}} + fresh_client.reload_credentials() + assert fresh_client.user_auth_token == 'TOKEN-1' + assert fresh_client.app_id is None + assert fresh_client.is_authenticated() is False + + +# --------------------------------------------------------------------------- +# Sync scenario — the actual reported bug +# --------------------------------------------------------------------------- + + +class TestTwoInstanceSync: + """Reproduce the Foxxify scenario: instance A logs in (writes to + config), instance B reads stale state until reload_credentials is + called.""" + + def test_second_instance_unaware_until_reload(self, qobuz_client_module): + module, config = qobuz_client_module + instance_a = module.QobuzClient() + instance_b = module.QobuzClient() + + # Instance A "logs in" — directly mutate the way login() would + instance_a.app_id = 'APP-A' + instance_a.app_secret = 'SECRET-A' + instance_a.user_auth_token = 'TOKEN-A' + instance_a._save_session() + + # Instance B is still in the dark. + assert instance_b.user_auth_token is None + assert instance_b.is_authenticated() is False + + # Sync. + instance_b.reload_credentials() + + assert instance_b.user_auth_token == 'TOKEN-A' + assert instance_b.is_authenticated() is True + assert instance_b.session.headers.get('X-User-Auth-Token') == 'TOKEN-A' diff --git a/tests/test_replaygain_summary_parse.py b/tests/test_replaygain_summary_parse.py new file mode 100644 index 00000000..094344ef --- /dev/null +++ b/tests/test_replaygain_summary_parse.py @@ -0,0 +1,198 @@ +"""Pin the ReplayGain analysis fix. + +User report: every track in a downloaded album got the same +``replaygain_track_gain`` of ``+52.00 dB`` after post-processing. +Smoking gun: ``-18 (RG2 reference) - (-70.0) = +52.00``. Every track's +first ebur128 measurement window reads ~-70 LUFS because the first +window covers the silent intro / encoder padding. + +The old code used ``re.search('I:\\s+...')`` which returns the FIRST +match — capturing that initial -70 LUFS reading instead of the final +integrated value from the Summary block. + +These tests use representative ffmpeg ebur128 output (per-window +progress + final Summary block) to pin: parser anchors to the +Summary, ignores per-window partials, and falls back gracefully when +Summary is absent. +""" + +from __future__ import annotations + +import re +import subprocess +from unittest.mock import patch + +import pytest + +from core.replaygain import analyze_track + + +# --------------------------------------------------------------------------- +# Fabricated ebur128 stderr samples +# --------------------------------------------------------------------------- + +_REAL_EBUR128_STDERR = """ +[Parsed_ebur128_0 @ 0x000001] Summary: +[Parsed_ebur128_0 @ 0x000001] t: 0.500000 TARGET:-23 LUFS M: -70.0 S:-70.0 I: -70.0 LUFS LRA: 0.0 LU FTPK: -70.0 dBFS TPK: -70.0 dBFS +[Parsed_ebur128_0 @ 0x000001] t: 1.000000 TARGET:-23 LUFS M: -50.0 S:-60.0 I: -60.0 LUFS LRA: 0.0 LU FTPK: -50.0 dBFS TPK: -50.0 dBFS +[Parsed_ebur128_0 @ 0x000001] t: 1.500000 TARGET:-23 LUFS M: -20.0 S:-30.0 I: -25.0 LUFS LRA: 0.0 LU FTPK: -2.5 dBFS TPK: -2.5 dBFS +[Parsed_ebur128_0 @ 0x000001] t: 2.000000 TARGET:-23 LUFS M: -18.0 S:-20.0 I: -14.5 LUFS LRA: 1.5 LU FTPK: -0.4 dBFS TPK: -0.4 dBFS +[Parsed_ebur128_0 @ 0x000001] Summary: + + Integrated loudness: + I: -14.3 LUFS + Threshold: -24.3 LUFS + + Loudness range: + LRA: 3.2 LU + Threshold: -34.3 LUFS + LRA low: -16.5 LUFS + LRA high: -13.3 LUFS + + True peak: + Peak: -0.4 dBFS +[out#0/null @ 0x000002] video:0KiB audio:172KiB +""" + + +def _stub_ffmpeg(stderr_output: str): + """Patch subprocess.run to return a fake ffmpeg result with the + given stderr.""" + class _FakeResult: + def __init__(self): + self.stderr = stderr_output + self.returncode = 0 + return patch.object(subprocess, 'run', return_value=_FakeResult()) + + +# --------------------------------------------------------------------------- +# Headline regression: don't grab the first per-window reading +# --------------------------------------------------------------------------- + + +def test_parses_summary_lufs_not_first_per_window_reading(): + """The per-window stream contains 'I: -70.0 LUFS' (silent intro) + BEFORE the Summary block's 'I: -14.3 LUFS'. Parser must return + -14.3 (summary), NOT -70.0 (first per-window). + + This is the exact bug from the user's +52.00 dB report: + -18 RG2 reference - (-70.0) = +52.00 was the symptom.""" + with _stub_ffmpeg(_REAL_EBUR128_STDERR): + lufs, peak = analyze_track('/fake/path.flac') + + assert lufs == pytest.approx(-14.3, abs=0.01) + assert peak == pytest.approx(-0.4, abs=0.01) + + +def test_resulting_gain_is_realistic_not_plus_52(): + """Computed gain must be a normal real-world value (a few dB + range), NOT the symptomatic +52.00 dB the bug produced.""" + with _stub_ffmpeg(_REAL_EBUR128_STDERR): + lufs, _peak = analyze_track('/fake/path.flac') + gain = -18.0 - lufs # RG2 reference + assert -10.0 < gain < 10.0, f"Unrealistic gain {gain:+.2f} dB — bug regression" + + +# --------------------------------------------------------------------------- +# Different per-track values stay different +# --------------------------------------------------------------------------- + + +def _make_stderr(per_window_lufs: list[float], summary_lufs: float, summary_peak: float) -> str: + """Build an ebur128 stderr blob with controllable per-window and + summary values. Lets each test verify the summary is what gets + used regardless of what's in the per-window stream.""" + per_window = '\n'.join( + f"[Parsed_ebur128_0 @ 0x1] t: {(i + 1) * 0.5:.6f} TARGET:-23 LUFS " + f"M: {lufs:.1f} S:{lufs:.1f} I: {lufs:.1f} LUFS LRA: 0.0 LU " + f"FTPK: {lufs / 2:.1f} dBFS TPK: {lufs / 2:.1f} dBFS" + for i, lufs in enumerate(per_window_lufs) + ) + return f"""{per_window} +[Parsed_ebur128_0 @ 0x1] Summary: + + Integrated loudness: + I: {summary_lufs:.1f} LUFS + Threshold: -24.0 LUFS + + Loudness range: + LRA: 3.2 LU + + True peak: + Peak: {summary_peak:+.1f} dBFS +""" + + +def test_two_tracks_with_different_summaries_get_different_lufs(): + """Two simulated tracks with the SAME first per-window value (-70) + but DIFFERENT summary integrated loudness values. Old buggy parser + would report -70 for both. Fixed parser correctly reports the + distinct summary values.""" + track_a_stderr = _make_stderr([-70.0, -50.0, -20.0], summary_lufs=-14.3, summary_peak=-0.4) + track_b_stderr = _make_stderr([-70.0, -45.0, -10.0], summary_lufs=-7.8, summary_peak=-1.2) + + with _stub_ffmpeg(track_a_stderr): + lufs_a, _ = analyze_track('/fake/a.flac') + with _stub_ffmpeg(track_b_stderr): + lufs_b, _ = analyze_track('/fake/b.flac') + + assert lufs_a != lufs_b + assert lufs_a == pytest.approx(-14.3, abs=0.01) + assert lufs_b == pytest.approx(-7.8, abs=0.01) + + +def test_per_window_lufs_with_higher_value_doesnt_leak_into_summary(): + """If per-window readings include a value HIGHER than the summary + (a transient loud window), the parser must still return the + summary value, not the loudest per-window.""" + stderr = _make_stderr([-70.0, -5.0, -3.0], summary_lufs=-12.0, summary_peak=-0.5) + with _stub_ffmpeg(stderr): + lufs, _ = analyze_track('/fake/loud.flac') + assert lufs == pytest.approx(-12.0, abs=0.01) + + +# --------------------------------------------------------------------------- +# Defensive fallback when Summary block is absent +# --------------------------------------------------------------------------- + + +def test_falls_back_to_last_per_window_when_no_summary(): + """Some ffmpeg versions or truncated outputs may lack a Summary + block. Defensive fallback uses the LAST per-window reading (still + closer to the final integrated value than the first).""" + stderr = """ +[Parsed_ebur128_0 @ 0x1] t: 0.5 I: -70.0 LUFS LRA: 0.0 LU +[Parsed_ebur128_0 @ 0x1] t: 1.0 I: -25.0 LUFS LRA: 0.0 LU +[Parsed_ebur128_0 @ 0x1] t: 1.5 I: -14.5 LUFS LRA: 1.5 LU +""".strip() + + with _stub_ffmpeg(stderr): + lufs, _peak = analyze_track('/fake/no_summary.flac') + + # Last per-window reading, NOT the first + assert lufs == pytest.approx(-14.5, abs=0.01) + + +def test_raises_when_no_lufs_anywhere(): + """If ffmpeg output contains no LUFS values at all (parse failure + / wrong format), surface a clear RuntimeError so the caller can + decide whether to skip RG analysis.""" + stderr = "ffmpeg: garbled output, no LUFS data anywhere\n" + with _stub_ffmpeg(stderr): + with pytest.raises(RuntimeError, match='Could not parse'): + analyze_track('/fake/garbage.flac') + + +# --------------------------------------------------------------------------- +# Peak parsing — ensure it stays anchored to summary too +# --------------------------------------------------------------------------- + + +def test_peak_uses_summary_value_not_per_window_max(): + """Per-window output uses 'TPK:'/'FTPK:' labels; only the summary + uses 'Peak:'. Pin that the parser only catches the summary peak + even when per-window TPK values would be larger.""" + stderr = _make_stderr([-70.0, -45.0, -10.0], summary_lufs=-12.0, summary_peak=-0.4) + with _stub_ffmpeg(stderr): + _lufs, peak = analyze_track('/fake/peak.flac') + assert peak == pytest.approx(-0.4, abs=0.01) diff --git a/tests/test_soundcloud_client.py b/tests/test_soundcloud_client.py new file mode 100644 index 00000000..cb258f87 --- /dev/null +++ b/tests/test_soundcloud_client.py @@ -0,0 +1,755 @@ +"""Unit + integration tests for ``core/soundcloud_client.py``. + +The unit tests stub out ``yt_dlp`` so they run fast, deterministically, +and offline. They cover: search shape correctness, the artist/title +heuristic, the dispatch-key (``filename``) round trip, the download +state machine (success / failure / shutdown), the progress emitter, and +the cancel/clear ledger operations. + +The integration tests are gated behind ``-m soundcloud_live`` so they +don't run in CI by default. Run them locally to verify against real +SoundCloud: + + python -m pytest tests/test_soundcloud_client.py -m soundcloud_live -v -s + +They hit the public SoundCloud surface, so they require network access +and a working yt-dlp install. +""" + +from __future__ import annotations + +import asyncio +import os +import threading +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from core import soundcloud_client +from core.soundcloud_client import SoundcloudClient, _sanitize_filename +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult + + +# --------------------------------------------------------------------------- +# Module-level helpers +# --------------------------------------------------------------------------- + + +def test_sanitize_filename_strips_reserved_chars() -> None: + # Reserved chars become underscores; trailing punctuation gets stripped. + assert _sanitize_filename('Track / Name : "Bad" ?') == 'Track _ Name _ _Bad' + # Repeated underscores collapse, leading/trailing underscores trimmed. + assert _sanitize_filename('////track////') == 'track' + # Empty input still returns a usable filename, never an empty string. + assert _sanitize_filename('') == 'soundcloud_track' + + +def test_split_artist_from_title_uses_dash_separator() -> None: + artist, title = SoundcloudClient._split_artist_from_title( + "Daft Punk - Get Lucky", "officialdaftpunk" + ) + assert artist == "Daft Punk" + assert title == "Get Lucky" + + +def test_split_artist_from_title_falls_back_to_uploader_when_no_dash() -> None: + artist, title = SoundcloudClient._split_artist_from_title( + "Some Mix Title", "uploader_handle" + ) + assert artist == "uploader_handle" + assert title == "Some Mix Title" + + +def test_split_artist_from_title_rejects_too_short_artist_part() -> None: + """Things like "DJ - Mix" shouldn't get parsed as artist='DJ' / title='Mix' + when a 2-char artist is plausibly noise — but our threshold is >=2, so + "DJ" actually qualifies. This pins the boundary.""" + artist, title = SoundcloudClient._split_artist_from_title("a - hello", "uploader") + # 'a' is 1 char → fall through to uploader + assert artist == "uploader" + assert title == "a - hello" + + +def test_split_artist_from_title_handles_empty_input() -> None: + artist, title = SoundcloudClient._split_artist_from_title("", "fallback") + assert artist == "fallback" + assert title == "" + + +# --------------------------------------------------------------------------- +# Construction / availability gates +# --------------------------------------------------------------------------- + + +@pytest.fixture +def tmp_dl(tmp_path: Path) -> Path: + p = tmp_path / "downloads" + p.mkdir() + return p + + +def _wire_engine(client: SoundcloudClient) -> 'DownloadEngine': + """Phase C7: SoundCloud no longer owns its own active_downloads + dict — state lives on engine.DownloadEngine. Tests that + construct a bare client must wire an engine so download() can + dispatch and the client's query/cancel methods read from + somewhere. Returns the engine for tests that want to inspect + state directly.""" + from core.download_engine import DownloadEngine + engine = DownloadEngine() + client.set_engine(engine) + return engine + + +def test_is_available_when_yt_dlp_installed(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + # In our test env yt-dlp is installed (it's a hard dep) + assert client.is_available() is True + assert client.is_configured() is True + + +def test_is_available_false_when_yt_dlp_missing(tmp_dl: Path, monkeypatch) -> None: + monkeypatch.setattr(soundcloud_client, "yt_dlp", None) + client = SoundcloudClient(download_path=str(tmp_dl)) + assert client.is_available() is False + assert client.is_configured() is False + + +def test_is_authenticated_always_false_until_oauth_ships(tmp_dl: Path) -> None: + """Anonymous-only client. Pin the contract so a future OAuth tier + has to explicitly flip this.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + assert client.is_authenticated() is False + + +def test_download_path_created_on_construction(tmp_path: Path) -> None: + target = tmp_path / "nested" / "deeper" / "downloads" + assert not target.exists() + SoundcloudClient(download_path=str(target)) + assert target.exists() and target.is_dir() + + +def test_set_shutdown_check_assigns_callable(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + sentinel = lambda: True # noqa: E731 + client.set_shutdown_check(sentinel) + assert client.shutdown_check is sentinel + + +# --------------------------------------------------------------------------- +# Search +# --------------------------------------------------------------------------- + + +def _run(coro): + """Tiny helper — we have async methods to exercise but no async test runner.""" + return asyncio.run(coro) + + +def test_search_returns_empty_when_unavailable(tmp_dl: Path, monkeypatch) -> None: + monkeypatch.setattr(soundcloud_client, "yt_dlp", None) + client = SoundcloudClient(download_path=str(tmp_dl)) + tracks, albums = _run(client.search("anything")) + assert tracks == [] + assert albums == [] + + +def test_search_returns_empty_for_empty_or_invalid_query(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + assert _run(client.search("")) == ([], []) + assert _run(client.search(None)) == ([], []) # type: ignore[arg-type] + assert _run(client.search(42)) == ([], []) # type: ignore[arg-type] + + +def test_search_converts_yt_dlp_entries_to_track_results(tmp_dl: Path) -> None: + """Happy-path search: yt-dlp returns a list of entries, the client + converts each into a TrackResult, and the album list stays empty.""" + fake_entries = [ + { + 'id': '12345', + 'title': 'Daft Punk - Around the World', + 'uploader': 'daftpunkofficial', + 'url': 'https://soundcloud.com/daftpunk/around-the-world', + 'duration': 425.0, + }, + { + 'id': '67890', + 'title': 'Some DJ Mix Set', + 'uploader': 'somedj', + 'url': 'https://soundcloud.com/somedj/some-mix', + 'duration': 3600.0, + }, + ] + client = SoundcloudClient(download_path=str(tmp_dl)) + with patch.object(client, '_extract_search_entries', return_value=fake_entries): + tracks, albums = _run(client.search("daft punk")) + + assert albums == [] + assert len(tracks) == 2 + + # First entry: "Artist - Title" parsing kicked in + t1 = tracks[0] + assert isinstance(t1, TrackResult) + assert t1.username == 'soundcloud' + assert t1.artist == 'Daft Punk' + assert t1.title == 'Around the World' + assert t1.bitrate == 128 + assert t1.quality == 'mp3' + assert t1.duration == 425000 # ms + # Filename carries id + URL + display name for downstream dispatch + parts = t1.filename.split('||') + assert parts[0] == '12345' + assert parts[1] == 'https://soundcloud.com/daftpunk/around-the-world' + assert 'Daft Punk' in parts[2] + # Source metadata roundtrips + assert t1._source_metadata['source'] == 'soundcloud' + assert t1._source_metadata['track_id'] == '12345' + assert t1._source_metadata['permalink_url'] == 'https://soundcloud.com/daftpunk/around-the-world' + + # Second entry: no " - " in title, fall back to uploader as artist + t2 = tracks[1] + assert t2.artist == 'somedj' + assert t2.title == 'Some DJ Mix Set' + assert t2.duration == 3_600_000 + + +def test_search_skips_entries_without_url(tmp_dl: Path) -> None: + """No URL → can't download later → drop from results.""" + fake_entries = [ + {'id': '1', 'title': 'has url', 'url': 'https://soundcloud.com/x/y'}, + {'id': '2', 'title': 'no url'}, # gets skipped + {'id': '', 'title': 'empty id', 'url': 'https://soundcloud.com/x/z'}, # also skipped + ] + client = SoundcloudClient(download_path=str(tmp_dl)) + with patch.object(client, '_extract_search_entries', return_value=fake_entries): + tracks, _ = _run(client.search("any")) + assert len(tracks) == 1 + assert tracks[0]._source_metadata['track_id'] == '1' + + +def test_search_handles_yt_dlp_exception(tmp_dl: Path) -> None: + """yt-dlp can raise on rate limit / network blip — caller still gets + a clean empty list, never a raised exception.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + with patch.object(client, '_extract_search_entries', + side_effect=RuntimeError("network down")): + tracks, albums = _run(client.search("anything")) + assert tracks == [] + assert albums == [] + + +def test_search_handles_empty_entries(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + with patch.object(client, '_extract_search_entries', return_value=[]): + tracks, _ = _run(client.search("nothing")) + assert tracks == [] + + +def test_search_handles_malformed_entries_individually(tmp_dl: Path) -> None: + """One bad entry shouldn't poison the entire result set.""" + fake_entries = [ + {'id': '1', 'title': 'good', 'url': 'https://x/1'}, + # Missing all required fields → conversion returns None → skipped + {'something': 'weird'}, + {'id': '2', 'title': 'also good', 'url': 'https://x/2'}, + ] + client = SoundcloudClient(download_path=str(tmp_dl)) + with patch.object(client, '_extract_search_entries', return_value=fake_entries): + tracks, _ = _run(client.search("any")) + assert len(tracks) == 2 + + +# --------------------------------------------------------------------------- +# Download orchestration +# --------------------------------------------------------------------------- + + +def test_download_rejects_invalid_filename_format(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + # No || separator + assert _run(client.download('soundcloud', 'broken')) is None + + +def test_download_starts_thread_and_returns_id(tmp_dl: Path) -> None: + """Verify the contract: returns a download_id, engine record is + populated, the worker drives state to terminal.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + completed_path = tmp_dl / "track.mp3" + completed_path.write_bytes(b"x" * (200 * 1024)) # > MIN_AUDIO_SIZE + + with patch.object(client, '_download_sync', return_value=str(completed_path)): + download_id = _run(client.download( + 'soundcloud', + '999||https://soundcloud.com/x/y||Display Name', + file_size=0, + )) + + assert download_id is not None + deadline = time.time() + 2 + while time.time() < deadline: + record = engine.get_record('soundcloud', download_id) + if record and record['state'] == 'Completed, Succeeded': + break + time.sleep(0.05) + + info = engine.get_record('soundcloud', download_id) + assert info['state'] == 'Completed, Succeeded' + assert info['progress'] == 100.0 + assert info['file_path'] == str(completed_path) + assert info['username'] == 'soundcloud' + + +def test_download_thread_marks_failed_when_sync_returns_none(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + with patch.object(client, '_download_sync', return_value=None): + download_id = _run(client.download( + 'soundcloud', + '1||https://soundcloud.com/x/y||name', + )) + deadline = time.time() + 2 + while time.time() < deadline: + record = engine.get_record('soundcloud', download_id) + if record and record['state'] == 'Errored': + break + time.sleep(0.05) + assert engine.get_record('soundcloud', download_id)['state'] == 'Errored' + + +def test_download_thread_does_not_clobber_cancelled_state(tmp_dl: Path) -> None: + """If a user cancels mid-download and the sync function then returns + None, the worker must NOT overwrite the explicit Cancelled state + with a generic Errored state. The legacy per-client thread had + this guard; engine.worker._mark_terminal preserves it for every + source via a single check.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + + def _slow_sync(download_id, *_): + time.sleep(0.05) + engine.update_record('soundcloud', download_id, {'state': 'Cancelled'}) + return None + + with patch.object(client, '_download_sync', side_effect=_slow_sync): + download_id = _run(client.download('soundcloud', '1||u||n')) + + deadline = time.time() + 2 + while time.time() < deadline: + record = engine.get_record('soundcloud', download_id) + if record and record['state'] in ('Cancelled', 'Errored'): + break + time.sleep(0.05) + assert engine.get_record('soundcloud', download_id)['state'] == 'Cancelled' + + +# --------------------------------------------------------------------------- +# yt-dlp interaction (download_sync) +# --------------------------------------------------------------------------- + + +class _FakeYDL: + """Minimal stand-in for yt_dlp.YoutubeDL used to exercise download_sync.""" + + def __init__(self, opts): + self.opts = opts + self.last_url = None + self.fake_info = {'id': 'abc', 'title': 'fake', 'ext': 'mp3'} + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def extract_info(self, url, download=False): + self.last_url = url + if download: + # Write a fake audio file to the resolved path + resolved = self.prepare_filename(self.fake_info) + Path(resolved).parent.mkdir(parents=True, exist_ok=True) + Path(resolved).write_bytes(b"y" * (200 * 1024)) + return self.fake_info + + def prepare_filename(self, info): + # Simulate yt-dlp's outtmpl substitution + template = self.opts['outtmpl'] + return template.replace('%(ext)s', info.get('ext', 'mp3')) + + +def test_download_sync_writes_file_and_returns_path(tmp_dl: Path, monkeypatch) -> None: + fake_yt_dlp = SimpleNamespace( + YoutubeDL=_FakeYDL, + utils=SimpleNamespace(DownloadError=Exception), + ) + monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp) + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + + engine.add_record('soundcloud', 'dl1', { + 'id': 'dl1', 'filename': '', 'username': 'soundcloud', + 'state': 'Initializing', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + 'track_id': 'abc', 'permalink_url': 'u', 'display_name': 'My Track', + 'file_path': None, + }) + + result = client._download_sync('dl1', 'https://soundcloud.com/x/y', 'My Track') + assert result is not None + assert os.path.exists(result) + assert os.path.getsize(result) > 100 * 1024 + + +def test_download_sync_rejects_too_small_file(tmp_dl: Path, monkeypatch) -> None: + """Files under MIN_AUDIO_SIZE_BYTES indicate yt-dlp got a preview + snippet or junk response; reject and clean up.""" + + class _TinyYDL(_FakeYDL): + def __init__(self, opts): + super().__init__(opts) + self.fake_info = {'id': 'tiny', 'title': 'tiny', 'ext': 'mp3'} + + def extract_info(self, url, download=False): + self.last_url = url + if download: + resolved = self.prepare_filename(self.fake_info) + Path(resolved).parent.mkdir(parents=True, exist_ok=True) + Path(resolved).write_bytes(b"y" * 500) # Too small + return self.fake_info + + fake_yt_dlp = SimpleNamespace( + YoutubeDL=_TinyYDL, + utils=SimpleNamespace(DownloadError=Exception), + ) + monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp) + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + + engine.add_record('soundcloud', 'dl2', { + 'id': 'dl2', 'filename': '', 'username': 'soundcloud', + 'state': 'Initializing', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + 'track_id': 'tiny', 'permalink_url': 'u', 'display_name': 'Tiny', + 'file_path': None, + }) + result = client._download_sync('dl2', 'https://soundcloud.com/x/y', 'Tiny') + assert result is None + # File got cleaned up after rejection + target = tmp_dl / "Tiny.mp3" + assert not target.exists() + + +def test_download_sync_handles_yt_dlp_raising(tmp_dl: Path, monkeypatch) -> None: + """yt-dlp can raise DownloadError or any other exception. download_sync + should surface a clean None instead of propagating.""" + class _BoomYDL: + def __init__(self, opts): + pass + def __enter__(self): + return self + def __exit__(self, *args): + return False + def extract_info(self, *a, **kw): + raise RuntimeError("boom") + def prepare_filename(self, info): + return "" + + fake_yt_dlp = SimpleNamespace( + YoutubeDL=_BoomYDL, + utils=SimpleNamespace(DownloadError=Exception), + ) + monkeypatch.setattr(soundcloud_client, "yt_dlp", fake_yt_dlp) + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + + engine.add_record('soundcloud', 'dl3', { + 'id': 'dl3', 'filename': '', 'username': 'soundcloud', + 'state': 'Initializing', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + }) + assert client._download_sync('dl3', 'https://soundcloud.com/x/y', 'Boom') is None + + +def test_download_sync_returns_none_when_yt_dlp_unavailable(tmp_dl: Path, monkeypatch) -> None: + monkeypatch.setattr(soundcloud_client, "yt_dlp", None) + client = SoundcloudClient(download_path=str(tmp_dl)) + assert client._download_sync('any', 'u', 'name') is None + + +# --------------------------------------------------------------------------- +# Progress emitter +# --------------------------------------------------------------------------- + + +def test_update_download_progress_populates_ledger(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + engine.add_record('soundcloud', 'p1', { + 'id': 'p1', 'filename': '', 'username': 'soundcloud', + 'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + }) + speed_start = time.time() - 1.0 # 1 second ago + client._update_download_progress('p1', downloaded=512_000, total=1_024_000, + speed_start=speed_start) + info = engine.get_record('soundcloud', 'p1') + assert info['transferred'] == 512_000 + assert info['size'] == 1_024_000 + assert 49.0 <= info['progress'] <= 51.0 + assert info['speed'] > 0 + assert info['time_remaining'] is not None + assert 0 < info['time_remaining'] < 5 + + +def test_update_download_progress_caps_at_99_9(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + engine.add_record('soundcloud', 'p2', { + 'id': 'p2', 'filename': '', 'username': 'soundcloud', + 'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + }) + client._update_download_progress('p2', downloaded=1_000_000, + total=1_000_000, speed_start=time.time() - 1) + assert engine.get_record('soundcloud', 'p2')['progress'] == 99.9 + + +def test_update_download_progress_silently_skips_unknown_id(tmp_dl: Path) -> None: + """No-op if the download id isn't tracked — defensive against late hooks.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + _wire_engine(client) + # Should not raise + client._update_download_progress('does_not_exist', 100, 1000, time.time()) + + +def test_update_download_progress_fragmented_uses_fragment_count(tmp_dl: Path) -> None: + """HLS-aware progress: fragment 5 of 10 → ~50%, regardless of byte + estimate.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + engine.add_record('soundcloud', 'hls1', { + 'id': 'hls1', 'filename': '', 'username': 'soundcloud', + 'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + }) + client._update_download_progress_fragmented( + 'hls1', downloaded=512_000, fragment_index=5, fragment_count=10, + speed_start=time.time() - 1.0, + ) + info = engine.get_record('soundcloud', 'hls1') + assert 49.0 <= info['progress'] <= 51.0 + assert info['size'] == 1_024_000 # 512000 * (10/5) + assert info['time_remaining'] is not None and info['time_remaining'] > 0 + + +def test_update_download_progress_fragmented_caps_at_99_9(tmp_dl: Path) -> None: + """Final fragment shouldn't push percentage to 100.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + engine.add_record('soundcloud', 'hls2', { + 'id': 'hls2', 'filename': '', 'username': 'soundcloud', + 'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + }) + client._update_download_progress_fragmented( + 'hls2', downloaded=10_000_000, fragment_index=10, fragment_count=10, + speed_start=time.time() - 5.0, + ) + assert engine.get_record('soundcloud', 'hls2')['progress'] == 99.9 + + +def test_update_download_progress_fragmented_handles_zero_index(tmp_dl: Path) -> None: + """Defensive: fragment_index=0 on first callback → progress 0, + no crash.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + engine.add_record('soundcloud', 'hls3', { + 'id': 'hls3', 'filename': '', 'username': 'soundcloud', + 'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + }) + client._update_download_progress_fragmented( + 'hls3', downloaded=0, fragment_index=0, fragment_count=10, + speed_start=time.time(), + ) + assert engine.get_record('soundcloud', 'hls3')['progress'] == 0.0 + + +def test_update_download_progress_fragmented_silently_skips_unknown_id(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + # Must not raise + client._update_download_progress_fragmented( + 'unknown', downloaded=100, fragment_index=1, fragment_count=10, + speed_start=time.time(), + ) + + +# --------------------------------------------------------------------------- +# Status / cancel / clear +# --------------------------------------------------------------------------- + + +def test_get_all_downloads_returns_status_objects(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + engine.add_record('soundcloud', 's1', { + 'id': 's1', 'filename': 'f', 'username': 'soundcloud', + 'state': 'InProgress, Downloading', 'progress': 33.3, 'size': 1000, + 'transferred': 333, 'speed': 100, 'time_remaining': 7, + 'file_path': None, + }) + out = _run(client.get_all_downloads()) + assert len(out) == 1 + assert isinstance(out[0], DownloadStatus) + assert out[0].id == 's1' + assert out[0].progress == 33.3 + + +def test_get_download_status_returns_none_for_unknown(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + _wire_engine(client) + assert _run(client.get_download_status('nope')) is None + + +def test_cancel_download_marks_state(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + engine.add_record('soundcloud', 'c1', { + 'id': 'c1', 'filename': '', 'username': 'soundcloud', + 'state': 'InProgress, Downloading', 'progress': 50.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + }) + assert _run(client.cancel_download('c1')) is True + assert engine.get_record('soundcloud', 'c1')['state'] == 'Cancelled' + + +def test_cancel_download_with_remove_drops_entry(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + engine.add_record('soundcloud', 'c2', { + 'id': 'c2', 'filename': '', 'username': 'soundcloud', + 'state': 'InProgress, Downloading', 'progress': 0.0, 'size': 0, + 'transferred': 0, 'speed': 0, 'time_remaining': None, + }) + assert _run(client.cancel_download('c2', remove=True)) is True + assert engine.get_record('soundcloud', 'c2') is None + + +def test_cancel_download_returns_false_for_unknown(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + _wire_engine(client) + assert _run(client.cancel_download('not_real')) is False + + +def test_clear_completed_drops_terminal_entries_only(tmp_dl: Path) -> None: + """Terminal states get cleared; in-flight downloads survive.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + base = {'filename': '', 'username': 'soundcloud', 'progress': 0.0, + 'size': 0, 'transferred': 0, 'speed': 0, 'time_remaining': None} + engine.add_record('soundcloud', 'done', {**base, 'id': 'done', 'state': 'Completed, Succeeded'}) + engine.add_record('soundcloud', 'err', {**base, 'id': 'err', 'state': 'Errored'}) + engine.add_record('soundcloud', 'cnc', {**base, 'id': 'cnc', 'state': 'Cancelled'}) + engine.add_record('soundcloud', 'live', {**base, 'id': 'live', 'state': 'InProgress, Downloading'}) + + assert _run(client.clear_all_completed_downloads()) is True + assert engine.get_record('soundcloud', 'done') is None + assert engine.get_record('soundcloud', 'err') is None + assert engine.get_record('soundcloud', 'cnc') is None + assert engine.get_record('soundcloud', 'live') is not None + + +# --------------------------------------------------------------------------- +# Connection check +# --------------------------------------------------------------------------- + + +def test_check_connection_returns_false_when_unavailable(tmp_dl: Path, monkeypatch) -> None: + monkeypatch.setattr(soundcloud_client, "yt_dlp", None) + client = SoundcloudClient(download_path=str(tmp_dl)) + assert _run(client.check_connection()) is False + + +def test_check_connection_returns_true_on_successful_search(tmp_dl: Path) -> None: + client = SoundcloudClient(download_path=str(tmp_dl)) + + async def _fake_search(*_a, **_kw): + return ([MagicMock()], []) + + with patch.object(client, 'search', side_effect=_fake_search): + assert _run(client.check_connection()) is True + + +def test_check_connection_returns_false_when_search_raises(tmp_dl: Path) -> None: + """Connection check shouldn't propagate the underlying exception.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + + async def _boom(*_a, **_kw): + raise RuntimeError("network down") + + with patch.object(client, 'search', side_effect=_boom): + assert _run(client.check_connection()) is False + + +# --------------------------------------------------------------------------- +# Live integration tests (gated) +# --------------------------------------------------------------------------- +# Run with: python -m pytest tests/test_soundcloud_client.py -m soundcloud_live -v -s +# These hit real SoundCloud — network required, slow, and skip in default CI. + +pytestmark_live = pytest.mark.soundcloud_live + + +@pytestmark_live +def test_live_search_returns_real_results(tmp_dl: Path) -> None: + """Real query against SoundCloud's public search.""" + client = SoundcloudClient(download_path=str(tmp_dl)) + tracks, albums = _run(client.search("daft punk around the world")) + assert albums == [] + assert len(tracks) > 0 + # First result should at least have a title and a usable filename + t = tracks[0] + assert t.title or t.artist + assert '||' in t.filename + parts = t.filename.split('||') + assert parts[0] # track id + assert parts[1].startswith('https://') + + +@pytestmark_live +def test_live_download_a_known_public_track(tmp_dl: Path) -> None: + """Download a real public SoundCloud track end-to-end. This is the + headline smoke test — if this passes, the client genuinely works. + + We use a SoundCloud-Provided promotional track to avoid hammering + any specific creator's stats. If this URL ever 404s, swap it for + another reliably-public free track. + """ + client = SoundcloudClient(download_path=str(tmp_dl)) + engine = _wire_engine(client) + # Search-then-download flow: pick the first hit for a popular query + tracks, _ = _run(client.search("creative commons electronic music")) + assert tracks, "Live search returned no results" + first = tracks[0] + download_id = _run(client.download(first.username, first.filename)) + assert download_id is not None + + # Wait up to 60s for completion + deadline = time.time() + 60 + final_state = None + final_path = None + while time.time() < deadline: + info = engine.get_record('soundcloud', download_id) or {} + final_state = info.get('state') + final_path = info.get('file_path') + if final_state in {'Completed, Succeeded', 'Errored', 'Cancelled'}: + break + time.sleep(0.5) + + assert final_state == 'Completed, Succeeded', f"Live download didn't complete: {final_state}" + assert final_path is not None + assert os.path.exists(final_path) + assert os.path.getsize(final_path) > 100 * 1024 diff --git a/tests/test_tidal_auth_redirect_uri.py b/tests/test_tidal_auth_redirect_uri.py new file mode 100644 index 00000000..caf3a7b9 --- /dev/null +++ b/tests/test_tidal_auth_redirect_uri.py @@ -0,0 +1,203 @@ +"""Regression tests for Tidal /auth/tidal redirect_uri selection. + +Discord-reported (Foxxify): Tidal returned error 1002 ("Invalid +redirect URI") on every authentication attempt. The user had +``http://127.0.0.1:8889/tidal/callback`` registered in his Tidal +Developer Portal (matching the SoulSync UI default + docs), but +SoulSync was sending a network-IP-derived URI like +``http://192.168.x.x:8889/tidal/callback`` because the empty-config +fallback in /auth/tidal built the URI from ``request.host``. Tidal +compares strings exactly, so the URIs didn't match and authentication +failed before the user could even see Tidal's consent screen. + +These tests pin: +1. When ``tidal.redirect_uri`` is configured, that value is sent to + Tidal verbatim. +2. When the config is empty, SoulSync uses the constructor default + (``http://127.0.0.1:/tidal/callback``) — NOT a value built + from request.host. +3. Both cases work whether the user is accessing SoulSync via + localhost or a network IP (the access path is independent from the + authorize redirect_uri). +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch +from urllib.parse import parse_qs, urlparse + +import pytest + + +@pytest.fixture +def auth_route_client(monkeypatch: pytest.MonkeyPatch): + """Flask test client wired to render the Tidal auth flow without + spawning a real TidalClient or hitting external services.""" + # Force the "remote/docker" branch (route only renders the + # instructions page when one of those flags is true). + monkeypatch.setattr( + "os.path.exists", + lambda p: p == "/.dockerenv" or False, + ) + + fake_client = MagicMock() + fake_client.client_id = "fake-id" + fake_client.code_verifier = "v" * 40 + fake_client.code_challenge = "c" * 40 + fake_client.auth_url = "https://login.tidal.com/authorize" + # Constructor default that mirrors core/tidal_client.py:124. + fake_client.redirect_uri = "http://127.0.0.1:8889/tidal/callback" + fake_client._generate_pkce_challenge = MagicMock() + + with patch("core.tidal_client.TidalClient", return_value=fake_client): + with patch("web_server.add_activity_item"): + from web_server import app as flask_app + flask_app.config['TESTING'] = True + yield flask_app.test_client(), fake_client + + +def _extract_authorize_url(html: str) -> str | None: + """Pull the Tidal authorize URL out of the rendered instructions page.""" + import re + m = re.search(r'href="(https://login\.tidal\.com/authorize\?[^"]+)"', html) + return m.group(1) if m else None + + +def _extract_redirect_uri(html: str) -> str | None: + auth_url = _extract_authorize_url(html) + if not auth_url: + return None + parsed = urlparse(auth_url) + qs = parse_qs(parsed.query) + val = qs.get('redirect_uri', [None])[0] + return val + + +# --------------------------------------------------------------------------- +# Configured redirect_uri honored verbatim +# --------------------------------------------------------------------------- + + +class TestConfiguredRedirectUriIsHonored: + def test_localhost_config_sent_when_user_accesses_via_network_ip( + self, auth_route_client, monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The reported Foxxify scenario: user has 127.0.0.1:8889 + registered in Tidal portal AND set in SoulSync config, accesses + the Web UI from his network IP. The authorize URL must contain + the configured 127.0.0.1 URI, NOT a value built from + request.host (which would mismatch the portal and yield + Tidal error 1002).""" + client, _fake = auth_route_client + + from config.settings import config_manager + monkeypatch.setattr( + config_manager, "get", + lambda key, default=None: ( + "http://127.0.0.1:8889/tidal/callback" + if key == "tidal.redirect_uri" else default + ), + ) + + response = client.get("/auth/tidal", base_url="http://192.168.86.50:8008") + html = response.get_data(as_text=True) + + sent = _extract_redirect_uri(html) + assert sent == "http://127.0.0.1:8889/tidal/callback", ( + f"Configured redirect_uri must be sent verbatim — got {sent!r}" + ) + + def test_custom_port_config_sent_verbatim( + self, auth_route_client, monkeypatch: pytest.MonkeyPatch, + ) -> None: + """User with non-default port (e.g. SOULSYNC_TIDAL_CALLBACK_PORT=9999) + and matching portal registration.""" + client, _fake = auth_route_client + + from config.settings import config_manager + monkeypatch.setattr( + config_manager, "get", + lambda key, default=None: ( + "http://127.0.0.1:9999/tidal/callback" + if key == "tidal.redirect_uri" else default + ), + ) + + response = client.get("/auth/tidal", base_url="http://192.168.86.50:8008") + html = response.get_data(as_text=True) + + sent = _extract_redirect_uri(html) + assert sent == "http://127.0.0.1:9999/tidal/callback" + + def test_explicit_network_ip_config_also_honored( + self, auth_route_client, monkeypatch: pytest.MonkeyPatch, + ) -> None: + """User who deliberately registered their network IP with Tidal + and configured SoulSync to match — that registration must also + be honored, not overridden.""" + client, _fake = auth_route_client + + from config.settings import config_manager + monkeypatch.setattr( + config_manager, "get", + lambda key, default=None: ( + "http://192.168.86.50:8889/tidal/callback" + if key == "tidal.redirect_uri" else default + ), + ) + + response = client.get("/auth/tidal", base_url="http://192.168.86.50:8008") + html = response.get_data(as_text=True) + + sent = _extract_redirect_uri(html) + assert sent == "http://192.168.86.50:8889/tidal/callback" + + +# --------------------------------------------------------------------------- +# Empty config falls back to constructor default — NOT request.host +# --------------------------------------------------------------------------- + + +class TestEmptyConfigFallsBackToDefault: + def test_empty_config_uses_constructor_default_not_request_host( + self, auth_route_client, monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The actual Foxxify case (his SoulSync UI display showed the + default but config was empty because the placeholder never got + saved): empty config from a non-localhost request must NOT build + a network-IP redirect URI. The constructor default (matching + the documented portal registration) wins instead.""" + client, _fake = auth_route_client + + from config.settings import config_manager + monkeypatch.setattr( + config_manager, "get", + lambda key, default=None: ("" if key == "tidal.redirect_uri" else default), + ) + + response = client.get("/auth/tidal", base_url="http://192.168.86.50:8008") + html = response.get_data(as_text=True) + + sent = _extract_redirect_uri(html) + assert sent == "http://127.0.0.1:8889/tidal/callback", ( + f"Empty config must fall back to constructor default — got {sent!r}. " + "If this looks like 'http://192.168.x.x:8889/tidal/callback' the " + "request-host fallback got reintroduced and Foxxify's bug is back." + ) + + def test_empty_config_localhost_access_uses_default( + self, auth_route_client, monkeypatch: pytest.MonkeyPatch, + ) -> None: + client, _fake = auth_route_client + + from config.settings import config_manager + monkeypatch.setattr( + config_manager, "get", + lambda key, default=None: ("" if key == "tidal.redirect_uri" else default), + ) + + response = client.get("/auth/tidal", base_url="http://127.0.0.1:8008") + html = response.get_data(as_text=True) + + sent = _extract_redirect_uri(html) + assert sent == "http://127.0.0.1:8889/tidal/callback" diff --git a/tests/test_websocket_infrastructure.py b/tests/test_websocket_infrastructure.py index 392f80aa..4e17eead 100644 --- a/tests/test_websocket_infrastructure.py +++ b/tests/test_websocket_infrastructure.py @@ -52,10 +52,12 @@ class TestServiceStatus: status_events = [e for e in received if e['name'] == 'status:update'] assert len(status_events) >= 1 data = status_events[0]['args'][0] + assert 'metadata_source' in data assert 'spotify' in data assert 'media_server' in data assert 'soulseek' in data assert 'active_media_server' in data + assert 'source' in data['metadata_source'] assert 'authenticated' in data['spotify'] def test_status_matches_http(self, test_app, shared_state): @@ -73,6 +75,7 @@ class TestServiceStatus: assert len(status_events) >= 1 ws_data = status_events[0]['args'][0] + assert ws_data['metadata_source'] == http_data['metadata_source'] assert ws_data['spotify'] == http_data['spotify'] assert ws_data['media_server'] == http_data['media_server'] assert ws_data['soulseek'] == http_data['soulseek'] @@ -86,13 +89,13 @@ class TestServiceStatus: build = shared_state['build_status_payload'] # Mutate cache - status_cache['spotify']['source'] = 'itunes' + status_cache['metadata_source']['source'] = 'itunes' socketio.emit('status:update', build()) received = client.get_received() status_events = [e for e in received if e['name'] == 'status:update'] data = status_events[-1]['args'][0] - assert data['spotify']['source'] == 'itunes' + assert data['metadata_source']['source'] == 'itunes' # ========================================================================= diff --git a/web_server.py b/web_server.py index 831976c8..f9709dcc 100644 --- a/web_server.py +++ b/web_server.py @@ -5,7 +5,6 @@ import json import asyncio import requests import socket -import ipaddress import subprocess import platform import threading @@ -18,7 +17,7 @@ import collections import functools from contextlib import contextmanager from pathlib import Path -from urllib.parse import quote, urljoin, urlparse +from urllib.parse import urljoin, urlparse from concurrent.futures import ThreadPoolExecutor, as_completed from flask import Flask, render_template, request, jsonify, redirect, send_file, send_from_directory, Response, session, g, abort @@ -41,7 +40,7 @@ logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, system-info, update check, etc. # Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release. -_SOULSYNC_BASE_VERSION = "2.4.1" +_SOULSYNC_BASE_VERSION = "2.4.2" def _build_version_string(): """Append short commit hash to version when available (e.g. 2.35+abc1234).""" @@ -52,8 +51,8 @@ def _build_version_string(): result = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True, cwd=os.path.dirname(__file__) or '.') if result.returncode == 0: sha = result.stdout.strip() - except Exception: - pass + except Exception as e: + logger.debug("git rev-parse failed: %s", e) if sha: return f"{_SOULSYNC_BASE_VERSION}+{sha[:7]}" return _SOULSYNC_BASE_VERSION @@ -89,13 +88,15 @@ from plexapi.myplex import MyPlexAccount, MyPlexPinLogin from core.jellyfin_client import JellyfinClient from core.navidrome_client import NavidromeClient from core.soulseek_client import SoulseekClient -from core.download_orchestrator import DownloadOrchestrator +from core.download_orchestrator import DownloadOrchestrator, set_download_orchestrator from core.tidal_client import TidalClient # Added import for Tidal from core.matching_engine import MusicMatchingEngine from core.database_update_worker import DatabaseUpdateWorker from core.web_scan_manager import WebScanManager from core.metadata.cache import get_metadata_cache from core.metadata import registry as metadata_registry +from core.metadata import is_internal_image_host +from core.metadata import normalize_image_url as fix_artist_image_url from core.metadata.registry import ( clear_cached_metadata_client, get_metadata_source_label, @@ -103,6 +104,12 @@ from core.metadata.registry import ( get_spotify_disconnect_source, register_runtime_clients, ) +from core.metadata.status import ( + get_status_snapshot as get_metadata_status_snapshot, + get_spotify_status, + publish_spotify_status, + invalidate_metadata_status_caches, +) from core.imports.context import ( get_import_clean_album, get_import_clean_title, @@ -380,8 +387,8 @@ def _set_profile_context(): session.pop('profile_id', None) from flask import jsonify as _jsonify return _jsonify({"error": "profile_required", "message": "Profile no longer exists"}), 401 - except Exception: - pass # DB error — don't block requests, use the session value + except Exception as e: + logger.debug("profile session validate: %s", e) g.profile_id = pid @@ -408,8 +415,8 @@ def _log_slow_request(response): response.status_code, elapsed_ms, ) - except Exception: - pass + except Exception as e: + logger.debug("slow request log failed: %s", e) return response @@ -508,8 +515,8 @@ def check_download_permission(): profile = get_database().get_profile(pid) if profile and not profile.get('can_download', True): return jsonify({'success': False, 'error': 'Downloads are disabled for this profile.'}), 403 - except Exception: - pass # DB error — don't block + except Exception as e: + logger.debug("download permission check: %s", e) return None # --- Docker Helper Functions --- @@ -563,7 +570,7 @@ IS_SHUTTING_DOWN = False # Each client is initialized independently so one failure doesn't take down everything. # Previously, a single exception set ALL clients to None, breaking the entire app. logger.info("Initializing SoulSync services for Web UI...") -spotify_client = plex_client = jellyfin_client = navidrome_client = soulsync_library_client = soulseek_client = tidal_client = matching_engine = sync_service = web_scan_manager = None +spotify_client = download_orchestrator = tidal_client = matching_engine = sync_service = web_scan_manager = media_server_engine = None try: spotify_client = get_spotify_client() @@ -571,33 +578,70 @@ try: except Exception as e: logger.error(f" Spotify client failed to initialize: {e}") -try: - plex_client = PlexClient() - logger.info(" Plex client initialized") -except Exception as e: - logger.error(f" Plex client failed to initialize: {e}") -try: - jellyfin_client = JellyfinClient() - logger.info(" Jellyfin client initialized") -except Exception as e: - logger.error(f" Jellyfin client failed to initialize: {e}") +def _safe_init_media_client(factory, name): + """Build a media-server client, capturing per-server init failures + so one broken server doesn't take the engine down with it. -try: - navidrome_client = NavidromeClient() - logger.info(" Navidrome client initialized") -except Exception as e: - logger.error(f" Navidrome client failed to initialize: {e}") + Logs the exception class so the boot log distinguishes config errors + (ConnectionError, AuthenticationError) from genuine import / dependency + failures — the broad ``Exception`` catch is intentional (any failure + on this code path means the user can't use that server, and we want + the other three to keep working) but the log makes the cause + diagnosable.""" + try: + instance = factory() + logger.info(f" {name} client initialized") + return instance + except Exception as exc: + logger.error( + f" {name} client failed to initialize: {type(exc).__name__}: {exc}", + exc_info=True, + ) + return None + +# Build the MediaServerEngine. The engine OWNS the per-server client +# instances — no separate web_server.py globals (Cin's standard from +# the download refactor: drop redundant access paths). All callers go +# through media_server_engine.client(''). try: + from core.media_server.engine import MediaServerEngine, set_media_server_engine from core.soulsync_client import SoulSyncClient - soulsync_library_client = SoulSyncClient() - logger.info(" SoulSync library client initialized") + media_server_engine = MediaServerEngine(clients={ + 'plex': _safe_init_media_client(PlexClient, "Plex"), + 'jellyfin': _safe_init_media_client(JellyfinClient, "Jellyfin"), + 'navidrome': _safe_init_media_client(NavidromeClient, "Navidrome"), + 'soulsync': _safe_init_media_client(SoulSyncClient, "SoulSync library"), + }) + # Install as process-wide singleton so callers reaching via + # get_media_server_engine() see the same instance web_server.py + # constructs at boot. Matches the metadata + download engine + # patterns. + set_media_server_engine(media_server_engine) + logger.info(" Media server engine initialized") except Exception as e: - logger.error(f" SoulSync library client failed to initialize: {e}") + logger.error(f" Media server engine failed to initialize: {e}", exc_info=True) + # Fallback: empty engine so downstream `engine.client('plex')` + # returns None instead of AttributeError'ing on a None engine + # global. Pre-refactor each per-server client global had its own + # try/except so engine failure didn't take down dispatch sites; + # this preserves that resilience. If the fallback ALSO fails + # (e.g. the import itself broke), media_server_engine stays as + # the None initialized at the top of the module. + try: + from core.media_server.engine import MediaServerEngine, set_media_server_engine + media_server_engine = MediaServerEngine(clients={}) + set_media_server_engine(media_server_engine) + except Exception as fallback_exc: + logger.error(f" Empty-engine fallback also failed: {fallback_exc}", exc_info=True) try: - soulseek_client = DownloadOrchestrator() + download_orchestrator = DownloadOrchestrator() + # Install as the process-wide singleton so callers reaching for + # get_download_orchestrator() see the same instance web_server.py + # constructs at boot. Matches Cin's metadata engine pattern. + set_download_orchestrator(download_orchestrator) logger.info(" Download orchestrator initialized") except Exception as e: logger.error(f" Download orchestrator failed to initialize: {e}") @@ -615,35 +659,26 @@ except Exception as e: logger.error(f" Matching engine failed to initialize: {e}") try: - sync_service = PlaylistSyncService(spotify_client, plex_client, soulseek_client, jellyfin_client, navidrome_client) + sync_service = PlaylistSyncService(spotify_client, download_orchestrator, media_server_engine=media_server_engine) logger.info(" Playlist sync service initialized") except Exception as e: logger.error(f" Playlist sync service failed to initialize: {e}") -# Inject shutdown check callback into YouTube and Tidal clients (avoids circular imports) -if soulseek_client: - if hasattr(soulseek_client, 'youtube'): - soulseek_client.youtube.set_shutdown_check(lambda: IS_SHUTTING_DOWN) - logger.info(" Configured YouTube client shutdown callback") - if hasattr(soulseek_client, 'tidal'): - soulseek_client.tidal.set_shutdown_check(lambda: IS_SHUTTING_DOWN) - logger.info(" Configured Tidal download client shutdown callback") - if hasattr(soulseek_client, 'qobuz'): - soulseek_client.qobuz.set_shutdown_check(lambda: IS_SHUTTING_DOWN) - logger.info(" Configured Qobuz client shutdown callback") - if hasattr(soulseek_client, 'hifi'): - soulseek_client.hifi.set_shutdown_check(lambda: IS_SHUTTING_DOWN) - logger.info(" Configured HiFi client shutdown callback") +# Inject shutdown check callback into every download source that +# accepts one. Generic dispatch via the registry — no per-source +# attribute reaches needed. +if download_orchestrator and hasattr(download_orchestrator, 'registry'): + for _src_name, _src_client in download_orchestrator.registry.all_plugins(): + if _src_client is not None and hasattr(_src_client, 'set_shutdown_check'): + try: + _src_client.set_shutdown_check(lambda: IS_SHUTTING_DOWN) + logger.info(" Configured %s client shutdown callback", _src_name) + except Exception as _exc: + logger.warning(" %s set_shutdown_check failed: %s", _src_name, _exc) # Initialize web scan manager for automatic post-download scanning try: - media_clients = { - 'plex_client': plex_client, - 'jellyfin_client': jellyfin_client, - 'navidrome_client': navidrome_client, - 'soulsync_library_client': soulsync_library_client, - } - web_scan_manager = WebScanManager(media_clients, delay_seconds=60) + web_scan_manager = WebScanManager(media_server_engine, delay_seconds=60) logger.info(" Web scan manager initialized") except Exception as e: logger.error(f" Web scan manager failed to initialize: {e}") @@ -811,12 +846,10 @@ _idle_since = {} _IDLE_GRACE_SECONDS = 5 _status_cache = { - 'spotify': {'connected': False, 'authenticated': False, 'response_time': 0, 'source': 'itunes'}, 'media_server': {'connected': False, 'response_time': 0, 'type': None}, 'soulseek': {'connected': False, 'response_time': 0}, } _status_cache_timestamps: dict[str, float] = { - 'spotify': 0, 'media_server': 0, 'soulseek': 0, } @@ -1216,8 +1249,8 @@ def _register_automation_handlers(): 'added': str(added_count), 'removed': str(removed_count), }) - except Exception: - pass + except Exception as e: + logger.debug("playlist_synced automation emit failed: %s", e) else: logger.warning(f"[AUTOMATION] No changes: '{pl.get('name', '')}' (tracks={len(old_ids)})") _update_automation_progress(auto_id, @@ -1364,8 +1397,8 @@ def _register_automation_handlers(): 'status': 'skipped', 'reason': f'All {len(tracks_json)} tracks unchanged since last sync', } - except Exception: - pass # If we can't read last status, just run the sync + except Exception as e: + logger.debug("mirror sync last-status read: %s", e) _update_automation_progress(auto_id, progress=50, phase=f'Syncing "{pl["name"]}"', @@ -1822,8 +1855,8 @@ def _register_automation_handlers(): elif os.path.isdir(fp): _shutil.rmtree(fp) removed += 1 - except Exception: - pass + except Exception as e: + logger.debug("quarantine entry purge failed: %s", e) _update_automation_progress(automation_id, log_line=f'Removed {removed} quarantined items', log_type='success' if removed > 0 else 'info') return {'status': 'completed', 'removed': str(removed)} @@ -1928,8 +1961,8 @@ def _register_automation_handlers(): while len(existing) > max_backups: try: os.remove(existing.pop(0)) - except Exception: - pass + except Exception as e: + logger.debug("rolling backup cleanup failed: %s", e) _update_automation_progress(automation_id, log_line=f'Backup created: {size_mb}MB ({os.path.basename(backup_path)})', log_type='success') return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)} @@ -1991,9 +2024,9 @@ def _register_automation_handlers(): hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) soulseek_active = (dl_mode == 'soulseek' or (dl_mode == 'hybrid' and 'soulseek' in hybrid_order)) - # soulseek_client is a DownloadOrchestrator; the real client lives on - # .soulseek. Match the getattr pattern used at the other call sites. - slskd = getattr(soulseek_client, 'soulseek', None) if soulseek_client else None + # Reach the underlying SoulseekClient via the orchestrator's + # generic accessor. + slskd = download_orchestrator.client('soulseek') if download_orchestrator else None if not soulseek_active or not slskd or not slskd.base_url: _update_automation_progress(automation_id, log_line='Soulseek not active — skipped', log_type='skip') @@ -2003,7 +2036,7 @@ def _register_automation_handlers(): log_line='Auto-clear disabled in settings', log_type='skip') return {'status': 'skipped'} try: - success = run_async(soulseek_client.maintain_search_history_with_buffer( + success = run_async(download_orchestrator.maintain_search_history_with_buffer( keep_searches=50, trigger_threshold=200 )) if success: @@ -2039,7 +2072,7 @@ def _register_automation_handlers(): log_line='Skipped — downloads active', log_type='skip') return {'status': 'completed'} - run_async(soulseek_client.clear_all_completed_downloads()) + run_async(download_orchestrator.clear_all_completed_downloads()) if not has_post_processing: _sweep_empty_download_directories() _update_automation_progress(automation_id, @@ -2068,8 +2101,8 @@ def _register_automation_handlers(): elif os.path.isdir(fp): _shutil.rmtree(fp) q_removed += 1 - except Exception: - pass + except Exception as e: + logger.debug("quarantine entry purge failed: %s", e) steps.append(f'Quarantine: removed {q_removed} items') _update_automation_progress(automation_id, log_line=f'Quarantine: removed {q_removed} items', log_type='success' if q_removed else 'info') @@ -2094,7 +2127,7 @@ def _register_automation_handlers(): log_line='Download queue: skipped (active batches)', log_type='skip') else: try: - run_async(soulseek_client.clear_all_completed_downloads()) + run_async(download_orchestrator.clear_all_completed_downloads()) steps.append('Download queue: cleared') _update_automation_progress(automation_id, log_line='Download queue: cleared', log_type='success') @@ -2133,8 +2166,8 @@ def _register_automation_handlers(): for hidden in entries: try: os.remove(os.path.join(dirpath, hidden)) - except Exception: - pass + except Exception as e: + logger.debug("hidden file cleanup failed: %s", e) try: os.rmdir(dirpath) s_removed += 1 @@ -2152,7 +2185,7 @@ def _register_automation_handlers(): _update_automation_progress(automation_id, log_line='Search cleanup: disabled in settings', log_type='skip') else: - run_async(soulseek_client.maintain_search_history_with_buffer( + run_async(download_orchestrator.maintain_search_history_with_buffer( keep_searches=50, trigger_threshold=200 )) steps.append('Search history: cleaned') @@ -2284,7 +2317,7 @@ def _register_automation_handlers(): if automation_id: _update_automation_progress(automation_id, phase='Searching', log_line=f'Searching: {query}', log_type='info') - result = run_async(soulseek_client.search_and_download_best(query)) + result = run_async(download_orchestrator.search_and_download_best(query)) if result: if automation_id: _update_automation_progress(automation_id, @@ -2346,7 +2379,7 @@ try: app.register_blueprint(api_bp, url_prefix='/api/v1') app.soulsync = { 'spotify_client': spotify_client, - 'soulseek_client': soulseek_client, + 'download_orchestrator': download_orchestrator, 'tidal_client': tidal_client, 'matching_engine': matching_engine, 'config_manager': config_manager, @@ -2451,8 +2484,9 @@ def get_cached_transfer_data(): # First, get Soulseek downloads from API transfers_data = None - if not soulseek_known_down and soulseek_client and getattr(soulseek_client, 'soulseek', None) and soulseek_client.soulseek.base_url: - transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + _slsk = download_orchestrator.client("soulseek") if download_orchestrator else None + if not soulseek_known_down and _slsk and _slsk.base_url: + transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads')) if transfers_data: all_transfers = [] for user_data in transfers_data: @@ -2467,29 +2501,35 @@ def get_cached_transfer_data(): key = _make_context_key(transfer.get('username'), transfer.get('filename', '')) live_transfers_lookup[key] = transfer - # Also add non-Soulseek downloads (avoid redundant slskd call through orchestrator) + # Also add non-Soulseek downloads. Soulseek is excluded + # because slskd's transfers endpoint was already pulled + # above — without the exclude both fetch paths run. + # Every streaming source must appear here — task progress + # for in-flight downloads comes from this lookup. Missing a + # source = task.progress stays at 0 even when the + # underlying client knows the real percent. try: all_downloads = [] - for _dl_client in [soulseek_client.youtube, soulseek_client.tidal, soulseek_client.qobuz, - soulseek_client.hifi, soulseek_client.deezer_dl, soulseek_client.lidarr]: - if _dl_client: - try: - all_downloads.extend(run_async(_dl_client.get_all_downloads())) - except Exception: - pass + if download_orchestrator and hasattr(download_orchestrator, 'engine'): + try: + all_downloads = run_async( + download_orchestrator.engine.get_all_downloads(exclude=('soulseek',)) + ) + except Exception as e: + logger.debug("get_all_downloads failed: %s", e) for download in all_downloads: - key = _make_context_key(download.username, download.filename) - # Convert DownloadStatus to transfer dict format - live_transfers_lookup[key] = { - 'id': download.id, - 'filename': download.filename, - 'username': download.username, - 'state': download.state, - 'percentComplete': download.progress, - 'size': download.size, - 'bytesTransferred': download.transferred, - 'averageSpeed': download.speed, - } + key = _make_context_key(download.username, download.filename) + # Convert DownloadStatus to transfer dict format + live_transfers_lookup[key] = { + 'id': download.id, + 'filename': download.filename, + 'username': download.username, + 'state': download.state, + 'percentComplete': download.progress, + 'size': download.size, + 'bytesTransferred': download.transferred, + 'averageSpeed': download.speed, + } except Exception as e: logger.error(f"Could not fetch streaming source downloads: {e}") @@ -2934,13 +2974,13 @@ def _atexit_save_history(): try: from core.api_call_tracker import api_call_tracker api_call_tracker.save() - except Exception: + except Exception: # noqa: S110 — atexit handler, log handles may be closed pass def _atexit_shutdown(): try: _shutdown_runtime_components() - except Exception: + except Exception: # noqa: S110 — atexit handler, log handles may be closed pass @@ -3047,7 +3087,7 @@ def _build_prepare_stream_deps(): return _streaming_prepare.PrepareStreamDeps( config_manager=config_manager, - soulseek_client=soulseek_client, + download_orchestrator=download_orchestrator, stream_lock=stream_lock, project_root=os.path.dirname(os.path.abspath(__file__)), docker_resolve_path=docker_resolve_path, @@ -3406,8 +3446,8 @@ def _get_enrichment_status(): if key == 'spotify_enrichment': try: svc_data['daily_budget'] = worker._get_daily_budget_info() - except Exception: - pass + except Exception as e: + logger.debug("spotify daily budget read failed: %s", e) services[key] = svc_data else: @@ -3446,61 +3486,16 @@ def get_status(): current_time = time.time() active_server = config_manager.get_active_media_server() - # Test Spotify - with caching to avoid excessive API calls - if current_time - _status_cache_timestamps['spotify'] > STATUS_CACHE_TTL: - # Guard against spotify_client being None (partial init) - is_rate_limited = spotify_client.is_rate_limited() if spotify_client else False - rate_limit_info = spotify_client.get_rate_limit_info() if (spotify_client and is_rate_limited) else None - cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0 - spotify_session_active = spotify_client.is_spotify_authenticated() if spotify_client else False + metadata_status = get_metadata_status_snapshot(spotify_client=spotify_client) - # Read configured source once — no auth validation here, we do that explicitly below - configured_source = config_manager.get('metadata.fallback_source', 'deezer') or 'deezer' - - if is_rate_limited or cooldown_remaining > 0: - # During rate limit or post-ban cooldown, skip the auth probe entirely. - # Probing Spotify here would reset the rate limit timer. - music_source = 'deezer' if configured_source == 'spotify' else configured_source - spotify_response_time = 0 - else: - spotify_start = time.time() - spotify_connected = spotify_client.is_spotify_authenticated() if spotify_client else False - spotify_response_time = (time.time() - spotify_start) * 1000 - # Use configured source; fall back to deezer only if Spotify isn't authenticated - if configured_source == 'spotify': - music_source = 'spotify' if spotify_connected else 'deezer' - else: - music_source = configured_source - - _status_cache['spotify'] = { - 'connected': spotify_session_active, - 'authenticated': spotify_session_active, - 'response_time': round(spotify_response_time, 1), - 'source': music_source, - 'rate_limited': is_rate_limited, - 'rate_limit': rate_limit_info, - 'post_ban_cooldown': cooldown_remaining if cooldown_remaining > 0 else None - } - _status_cache_timestamps['spotify'] = current_time - # else: use cached value - - # Test media server - use EXISTING instances (they have internal caching) - # Media server clients already cache connection checks internally + # Test media server — engine reads active_server config + dispatches + # to the right client (with internal connection caching). Engine + # returns False safely if the active client is None / not registered. if current_time - _status_cache_timestamps['media_server'] > STATUS_CACHE_TTL: media_server_start = time.time() - media_server_status = False - if active_server == "plex" and plex_client: - # Use existing instance - has 30s internal connection cache - media_server_status = plex_client.is_connected() - elif active_server == "jellyfin" and jellyfin_client: - # Use existing instance - has internal connection caching - media_server_status = jellyfin_client.is_connected() - elif active_server == "navidrome" and navidrome_client: - # Use existing instance - media_server_status = navidrome_client.is_connected() - elif active_server == "soulsync": - # Standalone mode — always connected if Transfer folder exists - media_server_status = soulsync_library_client.is_connected() if soulsync_library_client else False + media_server_status = ( + media_server_engine.is_connected() if media_server_engine else False + ) media_server_response_time = (time.time() - media_server_start) * 1000 _status_cache['media_server'] = { 'connected': media_server_status, @@ -3517,8 +3512,11 @@ def get_status(): soulseek_relevant = (download_mode == 'soulseek' or (download_mode == 'hybrid' and 'soulseek' in hybrid_order)) - # Serverless sources (YouTube, HiFi, Qobuz) are always available - serverless_sources = ('youtube', 'hifi', 'qobuz', 'tidal', 'deezer_dl') + # Serverless sources (YouTube, HiFi, Qobuz, Tidal, Deezer, Lidarr, SoundCloud) + # don't depend on slskd being reachable — when one of these is the + # active source, surface "connected" without probing slskd so the + # dashboard / sidebar indicator stays green. + serverless_sources = ('youtube', 'hifi', 'qobuz', 'tidal', 'deezer_dl', 'lidarr', 'soundcloud') is_serverless = (download_mode in serverless_sources or (download_mode == 'hybrid' and hybrid_order and any(s in serverless_sources for s in hybrid_order))) @@ -3527,10 +3525,10 @@ def get_status(): if is_serverless: soulseek_status = True soulseek_response_time = 0 - elif soulseek_relevant and soulseek_client: + elif soulseek_relevant and download_orchestrator: soulseek_start = time.time() try: - soulseek_status = run_async(soulseek_client.check_connection()) + soulseek_status = run_async(download_orchestrator.check_connection()) except Exception: soulseek_status = False soulseek_response_time = (time.time() - soulseek_start) * 1000 @@ -3556,7 +3554,8 @@ def get_status(): active_dl_count += 1 status_data = { - 'spotify': _status_cache['spotify'], + 'metadata_source': metadata_status['metadata_source'], + 'spotify': metadata_status['spotify'], 'media_server': _status_cache['media_server'], 'soulseek': _status_cache['soulseek'], 'active_media_server': active_server, @@ -3942,7 +3941,7 @@ def _build_system_stats(): if soulseek_active and not soulseek_known_down: try: - transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads')) if transfers_data: for user_data in transfers_data: if 'directories' in user_data: @@ -4161,23 +4160,24 @@ def handle_settings(): # Reload service clients with new settings (guard against None from partial init) if spotify_client: spotify_client.reload_config() - if plex_client: - plex_client.server = None - if jellyfin_client: - jellyfin_client.reload_config() - if navidrome_client: - navidrome_client.reload_config() + if media_server_engine.client('plex'): + media_server_engine.client('plex').server = None + if media_server_engine.client('jellyfin'): + media_server_engine.client('jellyfin').reload_config() + if media_server_engine.client('navidrome'): + media_server_engine.client('navidrome').reload_config() # Reload orchestrator settings (download source mode, hybrid_primary, etc.) - if soulseek_client: - soulseek_client.reload_settings() + if download_orchestrator: + download_orchestrator.reload_settings() # Reload YouTube client settings (rate limiting, cookies) - if hasattr(soulseek_client, 'youtube'): - soulseek_client.youtube.reload_settings() + _yt = download_orchestrator.client("youtube") + if _yt: + _yt.reload_settings() # FIX: Re-instantiate the global tidal_client to pick up new settings try: tidal_client = TidalClient() - except Exception: - pass # Keep existing tidal_client if re-init fails + except Exception as e: + logger.debug("tidal client re-init: %s", e) # Reload enrichment worker clients for key-based services if lastfm_worker: lastfm_worker._init_client() @@ -4185,8 +4185,16 @@ def handle_settings(): genius_worker._init_client() if tidal_enrichment_worker: tidal_enrichment_worker.client = tidal_client + if 'spotify' in new_settings: + publish_spotify_status( + connected=False, + authenticated=False, + rate_limited=False, + rate_limit=None, + post_ban_cooldown=None, + ) # Invalidate status cache so next poll reflects new settings (e.g. fallback source change) - _status_cache_timestamps['spotify'] = 0 + invalidate_metadata_status_caches() logger.info("Service clients re-initialized with new settings.") return jsonify({"success": True, "message": "Settings saved successfully."}) except Exception as e: @@ -4196,9 +4204,9 @@ def handle_settings(): data = dict(config_manager.config_data) # Include which download sources are configured so the UI can auto-disable unconfigured ones try: - data['_source_status'] = soulseek_client.get_source_status() - except Exception: - pass + data['_source_status'] = download_orchestrator.get_source_status() + except Exception as e: + logger.debug("download source status read failed: %s", e) return jsonify(data) except Exception as e: return jsonify({"error": str(e)}), 500 @@ -4322,8 +4330,8 @@ def hydrabase_connect(): if _hydrabase_ws: try: _hydrabase_ws.close() - except: - pass + except Exception as _e: + logger.debug("hydrabase connect-existing close: %s", _e) ws = websocket.create_connection( url, header={"x-api-key": api_key}, @@ -4348,8 +4356,8 @@ def hydrabase_disconnect(): if _hydrabase_ws: try: _hydrabase_ws.close() - except: - pass + except Exception as e: + logger.debug("hydrabase disconnect close: %s", e) _hydrabase_ws = None config_manager.set('hydrabase.auto_connect', False) # Only disable dev mode if not using Hydrabase as a regular fallback source @@ -4420,8 +4428,8 @@ def hydrabase_send(): with _hydrabase_lock: try: _hydrabase_ws.close() - except: - pass + except Exception as _e: + logger.debug("hydrabase send close: %s", _e) _hydrabase_ws = None return jsonify({"success": False, "error": str(e)}), 500 @@ -4803,11 +4811,7 @@ def test_connection_endpoint(): if success: current_time = time.time() if service == 'spotify': - spotify_session_active = spotify_client.is_spotify_authenticated() if spotify_client else False - _status_cache['spotify']['connected'] = True - _status_cache['spotify']['authenticated'] = spotify_session_active - _status_cache['spotify']['source'] = _get_metadata_fallback_source() - _status_cache_timestamps['spotify'] = current_time + invalidate_metadata_status_caches() logger.info("Updated Spotify status cache after successful test") elif service in ['plex', 'jellyfin', 'navidrome', 'soulsync']: _status_cache['media_server']['connected'] = True @@ -4971,11 +4975,7 @@ def test_dashboard_connection_endpoint(): if success: current_time = time.time() if service == 'spotify': - spotify_session_active = spotify_client.is_spotify_authenticated() if spotify_client else False - _status_cache['spotify']['connected'] = True - _status_cache['spotify']['authenticated'] = spotify_session_active - _status_cache['spotify']['source'] = _get_metadata_fallback_source() - _status_cache_timestamps['spotify'] = current_time + invalidate_metadata_status_caches() logger.info("Updated Spotify status cache after successful dashboard test") elif service in ['plex', 'jellyfin', 'navidrome', 'soulsync']: _status_cache['media_server']['connected'] = True @@ -5129,8 +5129,12 @@ def clear_plex_library_preference(): from database.music_database import MusicDatabase db = MusicDatabase() db.set_preference('plex_music_library', '') - if plex_client: - plex_client.music_library = None + plex = media_server_engine.client('plex') + if plex: + plex.music_library = None + # Also clear all-libraries mode so a fresh "select library" + # flow doesn't inherit stale state. + plex._all_libraries_mode = False return jsonify({"success": True, "message": "Plex library preference cleared."}) except Exception as e: logger.error(f"Error clearing Plex library preference: {e}") @@ -5141,17 +5145,22 @@ def clear_plex_library_preference(): def get_plex_music_libraries(): """Get list of all available music libraries from Plex""" try: - libraries = plex_client.get_available_music_libraries() + libraries = media_server_engine.client('plex').get_available_music_libraries() # Get currently selected library from database.music_database import MusicDatabase db = MusicDatabase() selected_library = db.get_preference('plex_music_library') - # Get the currently active library name + # Get the currently active library name. In all-libraries mode + # ``music_library`` is None — surface a friendly label so the + # settings UI displays the active selection correctly. current_library = None - if plex_client.music_library: - current_library = plex_client.music_library.title + plex = media_server_engine.client('plex') + if plex.music_library: + current_library = plex.music_library.title + elif plex.is_all_libraries_mode(): + current_library = 'All Libraries (combined)' return jsonify({ "success": True, @@ -5173,7 +5182,7 @@ def select_plex_music_library(): if not library_name: return jsonify({"success": False, "error": "No library name provided"}), 400 - success = plex_client.set_music_library_by_name(library_name) + success = media_server_engine.client('plex').set_music_library_by_name(library_name) if success: add_activity_item("", "Library Selected", f"Plex music library set to: {library_name}", "Now") @@ -5189,7 +5198,7 @@ def select_plex_music_library(): def get_jellyfin_users(): """Get list of Jellyfin users that have music libraries""" try: - users = jellyfin_client.get_available_users() + users = media_server_engine.client('jellyfin').get_available_users() # Get currently selected user from database.music_database import MusicDatabase @@ -5198,9 +5207,9 @@ def get_jellyfin_users(): # Determine the current user name from user_id current_user = None - if jellyfin_client.user_id: + if media_server_engine.client('jellyfin').user_id: for u in users: - if u['id'] == jellyfin_client.user_id: + if u['id'] == media_server_engine.client('jellyfin').user_id: current_user = u['name'] break @@ -5224,7 +5233,7 @@ def select_jellyfin_user(): if not username: return jsonify({"success": False, "error": "No username provided"}), 400 - success = jellyfin_client.set_user_by_name(username) + success = media_server_engine.client('jellyfin').set_user_by_name(username) if success: add_activity_item("", "User Selected", f"Jellyfin user set to: {username}", "Now") @@ -5240,7 +5249,7 @@ def select_jellyfin_user(): def get_jellyfin_music_libraries(): """Get list of all available music libraries from Jellyfin""" try: - libraries = jellyfin_client.get_available_music_libraries() + libraries = media_server_engine.client('jellyfin').get_available_music_libraries() # Get currently selected library from database.music_database import MusicDatabase @@ -5249,10 +5258,10 @@ def get_jellyfin_music_libraries(): # Get the currently active library name (match Plex behavior) current_library = None - if jellyfin_client.music_library_id: + if media_server_engine.client('jellyfin').music_library_id: # Look up library name from ID for lib in libraries: - if lib['key'] == jellyfin_client.music_library_id: + if lib['key'] == media_server_engine.client('jellyfin').music_library_id: current_library = lib['title'] break @@ -5276,7 +5285,7 @@ def select_jellyfin_music_library(): if not library_name: return jsonify({"success": False, "error": "No library name provided"}), 400 - success = jellyfin_client.set_music_library_by_name(library_name) + success = media_server_engine.client('jellyfin').set_music_library_by_name(library_name) if success: add_activity_item("", "Library Selected", f"Jellyfin music library set to: {library_name}", "Now") @@ -5292,19 +5301,19 @@ def select_jellyfin_music_library(): def get_navidrome_music_folders(): """Get list of available music folders from Navidrome""" try: - if not navidrome_client: + if not media_server_engine.client('navidrome'): return jsonify({"success": False, "error": "Navidrome client not configured"}), 400 - folders = navidrome_client.get_music_folders() + folders = media_server_engine.client('navidrome').get_music_folders() from database.music_database import MusicDatabase db = MusicDatabase() selected_folder = db.get_preference('navidrome_music_folder') current_folder = None - if navidrome_client.music_folder_id: + if media_server_engine.client('navidrome').music_folder_id: for f in folders: - if f['key'] == navidrome_client.music_folder_id: + if f['key'] == media_server_engine.client('navidrome').music_folder_id: current_folder = f['title'] break @@ -5325,7 +5334,7 @@ def select_navidrome_music_folder(): data = request.get_json() folder_name = data.get('folder_name', '') - success = navidrome_client.set_music_folder_by_name(folder_name) + success = media_server_engine.client('navidrome').set_music_folder_by_name(folder_name) if success: if folder_name: @@ -5613,19 +5622,32 @@ def auth_tidal(): tidal_oauth_state["code_verifier"] = temp_tidal_client.code_verifier tidal_oauth_state["code_challenge"] = temp_tidal_client.code_challenge - # Use the user's configured redirect_uri from settings — don't override - # with request.host, which in Docker returns the container hostname + # Use the user's configured redirect_uri from settings, falling back + # to the constructor default (``http://127.0.0.1:/tidal/callback``). + # The settings UI displays the default as the placeholder, and SoulSync's + # docs tell users to register THAT URI with their Tidal Developer App + # — Tidal validates the redirect_uri sent in the authorize request + # against the one in the portal, so sending anything else (e.g. a + # network-IP variant built from request.host) returns Tidal error 1002 + # "Invalid redirect URI" and the user can't authenticate. + # + # Docker/remote-access workflow is preserved by the post-auth swap step + # in the instructions page below: SoulSync sends ``127.0.0.1:``, + # Tidal redirects the user's browser to that URI (which fails locally), + # the instructions tell the user to swap ``127.0.0.1`` for the host + # they're accessing SoulSync from, and the swapped URL hits the + # container's exposed callback port. Building the URI from request.host + # at authorize time used to skip the swap entirely but broke users + # who registered the documented default. configured_redirect = config_manager.get('tidal.redirect_uri', '') if configured_redirect: temp_tidal_client.redirect_uri = configured_redirect logger.info(f"Using configured Tidal redirect_uri: {configured_redirect}") else: - # Fallback: dynamically set based on request host (non-Docker local access) - request_host = request.host.split(':')[0] - if request_host not in ('127.0.0.1', 'localhost'): - dynamic_redirect = f"http://{request_host}:8889/tidal/callback" - temp_tidal_client.redirect_uri = dynamic_redirect - logger.info(f"Tidal redirect_uri set from request host: {dynamic_redirect}") + logger.info( + f"Using default Tidal redirect_uri (no config override): " + f"{temp_tidal_client.redirect_uri}" + ) # Store PKCE + redirect_uri for callback to use the same values with tidal_oauth_lock: @@ -5854,7 +5876,7 @@ def spotify_callback(): return _spotify_auth_result_page("Your personal Spotify account is now connected. You can close this window.", authenticated=True) if profile_client: profile_client._invalidate_auth_cache() - _status_cache_timestamps['spotify'] = 0 + invalidate_metadata_status_caches() add_activity_item("", "Spotify Auth Warning", f"Profile {profile_id_from_state} completed OAuth but Spotify did not confirm an authenticated session", "Now") return _spotify_auth_result_page( "Spotify authorization completed, but SoulSync could not confirm an authenticated Spotify session for this profile. You can close this window and try Authenticate again.", @@ -5890,7 +5912,7 @@ def spotify_callback(): _clear_rate_limit() spotify_client._invalidate_auth_cache() # Invalidate status cache so next poll picks up the new connection - _status_cache_timestamps['spotify'] = 0 + invalidate_metadata_status_caches() # Refresh enrichment worker's client so it picks up new auth if spotify_enrichment_worker and hasattr(spotify_enrichment_worker, 'client'): spotify_enrichment_worker.client.reload_config() @@ -5900,7 +5922,7 @@ def spotify_callback(): else: logger.warning("Spotify OAuth token exchange succeeded but authentication validation failed") spotify_client._invalidate_auth_cache() - _status_cache_timestamps['spotify'] = 0 + invalidate_metadata_status_caches() add_activity_item("", "Spotify Auth Warning", "OAuth completed, but Spotify did not confirm an authenticated session", "Now") return _spotify_auth_result_page( "Spotify authorization completed, but SoulSync could not confirm an authenticated Spotify session. You can close this window and try Authenticate again.", @@ -5929,16 +5951,7 @@ def spotify_disconnect(): source_label = get_metadata_source_label(active_source) if configured_source == 'spotify': config_manager.set('metadata.fallback_source', active_source) - _status_cache['spotify'] = { - 'connected': False, - 'authenticated': False, - 'response_time': 0, - 'source': active_source, - 'rate_limited': False, - 'rate_limit': None, - 'post_ban_cooldown': None - } - _status_cache_timestamps['spotify'] = time.time() + invalidate_metadata_status_caches() add_activity_item("", "Spotify Disconnected", f"Using {source_label} for metadata", "Now") return jsonify({ 'success': True, @@ -5956,15 +5969,20 @@ def spotify_disconnect(): def spotify_rate_limit_status(): """Get Spotify rate limit ban details""" try: - info = spotify_client.get_rate_limit_info() + info = get_spotify_status(spotify_client=spotify_client) + rate_limit_info = info.get('rate_limit') if info: - return jsonify({ - 'rate_limited': True, - 'remaining_seconds': info['remaining_seconds'], - 'retry_after': info['retry_after'], - 'endpoint': info['endpoint'], - 'expires_at': info['expires_at'] - }) + payload = { + 'rate_limited': bool(info.get('rate_limited')), + } + if rate_limit_info: + payload.update({ + 'remaining_seconds': rate_limit_info.get('remaining_seconds', 0), + 'retry_after': rate_limit_info.get('retry_after'), + 'endpoint': rate_limit_info.get('endpoint'), + 'expires_at': rate_limit_info.get('expires_at'), + }) + return jsonify(payload) return jsonify({'rate_limited': False}) except Exception as e: logger.error(f"Error getting Spotify rate limit status: {e}") @@ -6100,8 +6118,8 @@ def deezer_callback(): try: json_data = resp.json() access_token = json_data.get('access_token') - except Exception: - pass + except Exception as e: + logger.debug("deezer token json parse failed: %s", e) if not access_token: return f"

No Access Token

Deezer response: {resp.text[:200]}

", 400 @@ -6746,7 +6764,7 @@ def _build_search_deps(): spotify_client=spotify_client, hydrabase_client=hydrabase_client, hydrabase_worker=hydrabase_worker, - soulseek_client=soulseek_client, + download_orchestrator=download_orchestrator, fix_artist_image_url=fix_artist_image_url, is_hydrabase_active=_is_hydrabase_active, get_metadata_fallback_source=_get_metadata_fallback_source, @@ -6772,7 +6790,7 @@ def search_music(): add_activity_item("", "Search Started", f"'{query}'", "Now") try: - results = _search_basic.run_basic_soulseek_search(query, soulseek_client, run_async) + results = _search_basic.run_basic_soulseek_search(query, download_orchestrator, run_async) add_activity_item("", "Search Complete", f"'{query}' - {len(results)} results", "Now") return jsonify({"results": results}) except Exception as e: @@ -6827,7 +6845,7 @@ def enhanced_search_source(source_name): When the requested source's client isn't available (Spotify unauthed, Discogs missing token, Hydrabase disconnected, MusicBrainz import - failure, soulseek_client.youtube missing), returns plain JSON + failure, download_orchestrator.client("youtube") missing), returns plain JSON `{"artists":[],"albums":[],"tracks":[],"available":false}` to match the original endpoint contract. """ @@ -6877,7 +6895,7 @@ def enhanced_search_library_check(): data = request.get_json() or {} result = _search_library_check.check_library_presence( database=get_database(), - plex_client=plex_client, + plex_client=media_server_engine.client('plex') if media_server_engine else None, config_manager=config_manager, profile_id=get_current_profile_id(), albums=data.get('albums', []), @@ -6912,7 +6930,7 @@ def stream_enhanced_search_track(): album_name=album_name, duration_ms=duration_ms, config_manager=config_manager, - soulseek_client=soulseek_client, + download_orchestrator=download_orchestrator, matching_engine=matching_engine, run_async=run_async, ) @@ -7051,7 +7069,7 @@ def download_music_video(): def _progress(pct): _music_video_downloads[video_id]['progress'] = round(pct, 1) - final_path = soulseek_client.youtube.download_music_video(video_url, output_path, progress_callback=_progress) + final_path = download_orchestrator.client("youtube").download_music_video(video_url, output_path, progress_callback=_progress) if final_path and os.path.exists(final_path): _music_video_downloads[video_id]['status'] = 'completed' @@ -7110,7 +7128,7 @@ def start_download(): filename = track_data.get('filename') file_size = track_data.get('size', 0) - download_id = run_async(soulseek_client.download( + download_id = run_async(download_orchestrator.download( username, filename, file_size @@ -7159,13 +7177,13 @@ def start_download(): if not username or not filename: return jsonify({"error": "Missing username or filename."}), 400 - download_id = run_async(soulseek_client.download(username, filename, file_size)) + download_id = run_async(download_orchestrator.download(username, filename, file_size)) logger.info(f"Download ID returned: {download_id}") if download_id: # Register download for post-processing (simple transfer to /Transfer) context_key = _make_context_key(username, filename) - is_streaming_source = username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr') + is_streaming_source = username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') with matched_context_lock: matched_downloads_context[context_key] = { 'search_result': { @@ -7355,7 +7373,7 @@ def get_download_status(): A robust status checker that correctly finds completed files by searching the entire download directory with fuzzy matching, mirroring the logic from downloads.py. """ - if not soulseek_client: + if not download_orchestrator: return jsonify({"transfers": []}) try: @@ -7364,7 +7382,7 @@ def get_download_status(): soulseek_known_down = not _status_cache.get('soulseek', {}).get('connected', True) transfers_data = None if not soulseek_known_down: - transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + transfers_data = run_async(download_orchestrator._make_request('GET', 'transfers/downloads')) # Don't return early if no Soulseek transfers - YouTube/Tidal downloads need to be checked too! all_transfers = [] @@ -7432,9 +7450,9 @@ def get_download_status(): transfer_id = file_info.get('id') if transfer_id: try: - run_async(soulseek_client.cancel_download(str(transfer_id), username, remove=True)) - except Exception: - pass + run_async(download_orchestrator.cancel_download(str(transfer_id), username, remove=True)) + except Exception as e: + logger.debug("orphan transfer cancel failed: %s", e) _orphaned_download_keys.discard(context_key) continue # Skip normal post-processing either way @@ -7544,10 +7562,10 @@ def get_download_status(): # Also include YouTube/Tidal downloads in the response try: - all_streaming_downloads = run_async(soulseek_client.get_all_downloads()) + all_streaming_downloads = run_async(download_orchestrator.get_all_downloads()) for download in all_streaming_downloads: - if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): + if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): source_label = download.username.title() # Convert DownloadStatus to transfer format that frontend expects streaming_transfer = { @@ -7676,7 +7694,7 @@ def cancel_download(): return jsonify({"success": False, "error": "Missing download_id or username."}), 400 try: - success = _downloads_cancel.cancel_single_download(soulseek_client, run_async, download_id, username) + success = _downloads_cancel.cancel_single_download(download_orchestrator, run_async, download_id, username) if success: return jsonify({"success": True, "message": "Download cancelled."}) return jsonify({"success": False, "error": "Failed to cancel download via slskd."}), 500 @@ -7690,7 +7708,7 @@ def cancel_all_downloads(): """Cancel all active downloads from slskd, then clear completed ones.""" try: success, msg = _downloads_cancel.cancel_all_active( - soulseek_client, run_async, _sweep_empty_download_directories, + download_orchestrator, run_async, _sweep_empty_download_directories, ) if success: return jsonify({"success": True, "message": msg}) @@ -7705,7 +7723,7 @@ def clear_finished_downloads(): """Clear all terminal (completed, cancelled, failed) downloads from slskd.""" try: success = _downloads_cancel.clear_finished_active( - soulseek_client, run_async, _sweep_empty_download_directories, + download_orchestrator, run_async, _sweep_empty_download_directories, ) if success: return jsonify({"success": True, "message": "Finished downloads cleared."}) @@ -7814,7 +7832,7 @@ def download_selected_candidate(task_id): batch['active_count'] = batch.get('active_count', 0) + 1 # Build a TrackResult-like candidate object - from core.soulseek_client import TrackResult + from core.download_plugins.types import TrackResult candidate = TrackResult( username=username, filename=filename, @@ -8044,9 +8062,9 @@ def test_automation_workflow(): results['media_clients'] = {'active_server': active_server} for client_name, client in [ - ('plex', plex_client), - ('jellyfin', jellyfin_client), - ('navidrome', navidrome_client) + ('plex', media_server_engine.client('plex')), + ('jellyfin', media_server_engine.client('jellyfin')), + ('navidrome', media_server_engine.client('navidrome')) ]: try: is_connected = client.is_connected() if client else False @@ -8092,7 +8110,7 @@ def clear_all_searches(): Clear all searches from slskd search history. """ try: - success = run_async(soulseek_client.clear_all_searches()) + success = run_async(download_orchestrator.clear_all_searches()) if success: add_activity_item("", "Search Cleanup", "All search history cleared manually", "Now") return jsonify({"success": True, "message": "All searches cleared."}) @@ -8112,7 +8130,7 @@ def maintain_search_history(): keep_searches = data.get('keep_searches', 50) trigger_threshold = data.get('trigger_threshold', 200) - success = run_async(soulseek_client.maintain_search_history_with_buffer( + success = run_async(download_orchestrator.maintain_search_history_with_buffer( keep_searches=keep_searches, trigger_threshold=trigger_threshold )) if success: @@ -8123,163 +8141,6 @@ def maintain_search_history(): except Exception as e: logger.error(f"Error maintaining search history: {e}") return jsonify({"success": False, "error": str(e)}), 500 - -def fix_artist_image_url(thumb_url): - """Convert media-server image URLs into browser-safe URLs.""" - if not thumb_url: - return None - - try: - # Check if it's a localhost URL or relative path that needs fixing - needs_fixing = ( - thumb_url.startswith('http://localhost:') or - thumb_url.startswith('https://localhost:') or - thumb_url.startswith('http://127.0.0.1:') or - thumb_url.startswith('https://127.0.0.1:') or - thumb_url.startswith('http://host.docker.internal:') or - thumb_url.startswith('https://host.docker.internal:') or - (thumb_url.startswith('http://') and _is_internal_image_host(thumb_url)) or - thumb_url.startswith('/library/') or # Plex relative paths - thumb_url.startswith('/Items/') or # Jellyfin relative paths - thumb_url.startswith('/api/') or # Old Navidrome API paths - thumb_url.startswith('/rest/') # Navidrome Subsonic API paths - ) - - if needs_fixing: - active_server = config_manager.get_active_media_server() - logger.debug(f"Fixing URL: {thumb_url}, Active server: {active_server}") - - if active_server == 'plex': - plex_config = config_manager.get_plex_config() - plex_base_url = plex_config.get('base_url', '') - plex_token = plex_config.get('token', '') - logger.info(f"Plex config - base_url: {plex_base_url}, token: {plex_token[:10]}...") - - if plex_base_url and plex_token: - # Extract the path from URL - if thumb_url.startswith('/library/'): - # Already a path - path = thumb_url - else: - # Full localhost URL, extract path - from urllib.parse import urlparse - parsed = urlparse(thumb_url) - path = parsed.path - - # Construct proper Plex URL with token - fixed_url = f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}" - logger.info(f"Fixed URL: {fixed_url}") - return _browser_safe_image_url(fixed_url) - - elif active_server == 'jellyfin': - jellyfin_config = config_manager.get_jellyfin_config() - jellyfin_base_url = jellyfin_config.get('base_url', '') - jellyfin_token = jellyfin_config.get('api_key', '') - logger.info(f"Jellyfin config - base_url: {jellyfin_base_url}, token: {jellyfin_token[:10] if jellyfin_token else 'None'}...") - - if jellyfin_base_url: - # Extract the path from URL - if thumb_url.startswith('/Items/') or thumb_url.startswith('/api/'): - # Already a path - path = thumb_url - else: - # Full localhost URL, extract path - from urllib.parse import urlparse - parsed = urlparse(thumb_url) - path = parsed.path - - # Construct proper Jellyfin URL with token - if jellyfin_token: - separator = '&' if '?' in path else '?' - fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}" - else: - fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}" - logger.info(f"Fixed URL: {fixed_url}") - return _browser_safe_image_url(fixed_url) - - elif active_server == 'navidrome': - navidrome_config = config_manager.get_navidrome_config() - navidrome_base_url = navidrome_config.get('base_url', '') - navidrome_username = navidrome_config.get('username', '') - navidrome_password = navidrome_config.get('password', '') - logger.info(f"Navidrome config - base_url: {navidrome_base_url}, username: {navidrome_username}") - - if navidrome_base_url and navidrome_username and navidrome_password: - # Extract the path from URL - if thumb_url.startswith('/rest/'): - # Already a Subsonic API path - path = thumb_url - else: - # Full localhost URL, extract path - from urllib.parse import urlparse - parsed = urlparse(thumb_url) - path = parsed.path - - # Generate Subsonic API authentication - import hashlib - import secrets - salt = secrets.token_hex(6) - token = hashlib.md5((navidrome_password + salt).encode()).hexdigest() - - # Add authentication parameters to the URL - separator = '&' if '?' in path else '?' - auth_params = f"u={navidrome_username}&t={token}&s={salt}&v=1.16.1&c=SoulSync&f=json" - - # Construct proper Navidrome Subsonic URL - fixed_url = f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}" - logger.info(f"Fixed URL: {fixed_url}") - return _browser_safe_image_url(fixed_url) - - logger.warning(f"No configuration found for {active_server} or unsupported server type") - - # Return a browser-safe URL even if no server-specific rebuild was possible. - return _browser_safe_image_url(thumb_url) - - except Exception as e: - logger.error(f"Error fixing image URL '{thumb_url}': {e}") - return _browser_safe_image_url(thumb_url) - - -def _is_internal_image_host(url: str) -> bool: - """Return True when an image URL points at a host the browser likely cannot reach directly.""" - try: - parsed = urlparse(url) - host = (parsed.hostname or '').strip('[]').lower() - if not host: - return False - - if host in {'localhost', '127.0.0.1', '::1', 'host.docker.internal'}: - return True - - # Single-label hosts are usually Docker service names or local LAN aliases. - if '.' not in host: - return True - - try: - ip = ipaddress.ip_address(host) - return ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_reserved - except ValueError: - return False - except Exception: - return False - - -def _browser_safe_image_url(url: str) -> str: - """Return a browser-safe image URL, proxying internal hosts through SoulSync.""" - if not url: - return url - - if url.startswith('/api/image-proxy?url='): - return url - - if url.startswith('http://') or url.startswith('https://'): - if _is_internal_image_host(url): - return f"/api/image-proxy?url={quote(url, safe='')}" - return url - - # Relative media-server paths should already have been expanded before this point. - return url - @app.route('/api/library/history') def get_library_history(): """Get persistent library history (downloads and server imports).""" @@ -8614,8 +8475,8 @@ def get_artist_detail(artist_id): except Exception: enrichment_coverage[svc] = 0 enrichment_coverage['total_tracks'] = total - except Exception: - pass + except Exception as e: + logger.debug("enrichment coverage build failed: %s", e) response_data = { "success": True, @@ -8749,6 +8610,108 @@ def get_artist_image(artist_id): logger.error(f"Error fetching artist image: {e}") return jsonify({"success": False, "image_url": None, "error": str(e)}) +@app.route('/api/artist//top-tracks', methods=['GET']) +def get_artist_top_tracks_endpoint(artist_id): + """Return an artist's top-N tracks via the primary metadata source. + + Issue #513: users want a "top X popular songs" path that doesn't pull + the entire discography. Spotify's `artist_top_tracks` endpoint and + Deezer's `/artist/{id}/top` both expose this; iTunes / Discogs / + MusicBrainz don't have popularity ranking, so this endpoint returns + `success=False` for those primary sources and the frontend falls back + to the existing Last.fm display-only sidebar. + + Resolves per-source artist IDs from the DB row (matching what + /discography already does) so a Spotify ID in the URL still works + when Deezer is primary, and vice versa. + """ + try: + primary_source = _get_metadata_fallback_source() + if primary_source not in ('spotify', 'deezer'): + return jsonify({ + 'success': False, + 'reason': 'unsupported_source', + 'source': primary_source, + 'tracks': [], + }) + + try: + limit = max(1, min(int(request.args.get('limit', 10)), 50)) + except (TypeError, ValueError): + limit = 10 + + # Per-source ID resolution from the DB — same pattern as + # /discography. Without this, the frontend's chosen ID type + # (Spotify, Deezer, iTunes, library DB id) decides which source + # can answer; we want the URL ID to be neutral. + resolved_id = artist_id + try: + _db = get_database() + _conn = _db._get_connection() + try: + _cur = _conn.cursor() + _cur.execute(""" + SELECT spotify_artist_id, deezer_id + FROM artists + WHERE id = ? + OR spotify_artist_id = ? + OR itunes_artist_id = ? + OR deezer_id = ? + OR musicbrainz_id = ? + LIMIT 1 + """, (artist_id, artist_id, artist_id, artist_id, artist_id)) + _row = _cur.fetchone() + if _row: + if primary_source == 'spotify' and _row['spotify_artist_id']: + resolved_id = str(_row['spotify_artist_id']) + elif primary_source == 'deezer' and _row['deezer_id']: + resolved_id = str(_row['deezer_id']) + finally: + _conn.close() + except Exception as e: + logger.debug("top-tracks per-source ID resolution failed: %s", e) + + tracks = [] + if primary_source == 'spotify': + if not spotify_client or not spotify_client.is_spotify_authenticated(): + return jsonify({ + 'success': False, + 'reason': 'spotify_not_authenticated', + 'source': 'spotify', + 'tracks': [], + }) + market = config_manager.get('spotify.market', 'US') or 'US' + tracks = spotify_client.get_artist_top_tracks(resolved_id, country=market, limit=limit) + else: # deezer + deezer_client = _get_deezer_client() + if not deezer_client: + return jsonify({ + 'success': False, + 'reason': 'deezer_unavailable', + 'source': 'deezer', + 'tracks': [], + }) + tracks = deezer_client.get_artist_top_tracks(resolved_id, limit=limit) + + if not tracks: + return jsonify({ + 'success': False, + 'reason': 'no_tracks_found', + 'source': primary_source, + 'tracks': [], + }) + + return jsonify({ + 'success': True, + 'source': primary_source, + 'resolved_artist_id': resolved_id, + 'tracks': tracks, + }) + except Exception as e: + logger.exception("Error fetching artist top tracks for %s", artist_id) + return jsonify({"success": False, "error": str(e), "tracks": []}), 500 + + @app.route('/api/artist//discography', methods=['GET']) def get_artist_discography(artist_id): """Get an artist's complete discography (albums and singles)""" @@ -8775,6 +8738,58 @@ def get_artist_discography(artist_id): from core.metadata.lookup import MetadataLookupOptions from core.metadata_service import get_artist_discography as _get_artist_discography + # Server-side per-source ID resolution. Look up the library row + # by ANY of the IDs the frontend might send: library DB id, + # spotify_artist_id, itunes_artist_id, deezer_id, or + # musicbrainz_id. Once matched, pull every stored provider ID + # and dispatch the right ID to each source via + # ``artist_source_ids``. Mirrors what the watchlist scanner + # already does. + # + # Without this, the frontend's ID choice fully decides which + # source can answer correctly: + # - sends DB id 194687 → Deezer accepts (wrong: it's a real + # Deezer ID for a different artist) + # - sends Spotify ID `1bDWGdIC...` → Deezer rejects → falls + # back to fuzzy name search → may pick wrong artist + # With server-side resolution, every source gets its OWN stored + # ID regardless of which one the URL carries. + artist_source_ids = {} + try: + _db = get_database() + _conn = _db._get_connection() + try: + _cur = _conn.cursor() + _cur.execute(""" + SELECT spotify_artist_id, itunes_artist_id, + deezer_id, musicbrainz_id + FROM artists + WHERE id = ? + OR spotify_artist_id = ? + OR itunes_artist_id = ? + OR deezer_id = ? + OR musicbrainz_id = ? + LIMIT 1 + """, (artist_id, artist_id, artist_id, artist_id, artist_id)) + _row = _cur.fetchone() + if _row: + if _row['spotify_artist_id']: + artist_source_ids['spotify'] = str(_row['spotify_artist_id']) + if _row['itunes_artist_id']: + artist_source_ids['itunes'] = str(_row['itunes_artist_id']) + if _row['deezer_id']: + artist_source_ids['deezer'] = str(_row['deezer_id']) + if _row['musicbrainz_id']: + artist_source_ids['musicbrainz'] = str(_row['musicbrainz_id']) + logger.info( + f"Discography: resolved per-source IDs for artist_id={artist_id} → " + f"{artist_source_ids}" + ) + finally: + _conn.close() + except Exception as _id_exc: + logger.debug(f"Could not resolve per-source artist IDs for {artist_id}: {_id_exc}") + discography = _get_artist_discography( artist_id, artist_name=artist_name, @@ -8784,6 +8799,7 @@ def get_artist_discography(artist_id): skip_cache=False, max_pages=0, limit=50, + artist_source_ids=artist_source_ids or None, ), ) @@ -8866,8 +8882,8 @@ def get_artist_discography(artist_id): if not artist_info.get('genres') and lib['genres']: try: artist_info['genres'] = json.loads(lib['genres']) - except Exception: - pass + except Exception as e: + logger.debug("genres json parse failed: %s", e) # Last.fm enrichment if lib.get('lastfm_bio'): artist_info['lastfm_bio'] = lib['lastfm_bio'] @@ -8878,8 +8894,8 @@ def get_artist_discography(artist_id): if lib.get('lastfm_tags'): try: artist_info['lastfm_tags'] = json.loads(lib['lastfm_tags']) if isinstance(lib['lastfm_tags'], str) else lib['lastfm_tags'] - except Exception: - pass + except Exception as e: + logger.debug("lastfm_tags json parse failed: %s", e) if lib.get('lastfm_url'): artist_info['lastfm_url'] = lib['lastfm_url'] if lib.get('genius_url'): @@ -9461,15 +9477,9 @@ def get_artist_enhanced_detail(artist_id): if album.get('thumb_url'): album['thumb_url'] = fix_artist_image_url(album['thumb_url']) - # Include server type for sync option + # Include server type for sync option — engine routes to active client. active_server = config_manager.get_active_media_server() - server_connected = False - if active_server == 'plex': - server_connected = plex_client.is_connected() - elif active_server == 'jellyfin': - server_connected = jellyfin_client.is_connected() - elif active_server == 'navidrome': - server_connected = navidrome_client.is_connected() + server_connected = media_server_engine.is_connected() if media_server_engine else False result['server_type'] = active_server if server_connected else None return jsonify(result) @@ -9557,14 +9567,44 @@ def _build_artist_quality_deps(): """Build the ArtistQualityDeps bundle from web_server.py globals on each call.""" from core.wishlist_service import get_wishlist_service as _get_ws + def _resolve_search_sources(): + """Mirror the Track Redownload modal's source list. Every + configured metadata source contributes to the parallel multi-source + search — Spotify (only when authenticated), iTunes, Deezer, plus + Discogs / Hydrabase when configured.""" + sources = [] + if spotify_client and spotify_client.is_authenticated(): + sources.append(('spotify', spotify_client)) + try: + sources.append(('itunes', _get_itunes_client())) + except Exception as e: + logger.debug("itunes client init failed: %s", e) + try: + sources.append(('deezer', _get_deezer_client())) + except Exception as e: + logger.debug("deezer client init failed: %s", e) + # Discogs needs an explicit token; only include when configured. + try: + _discogs_token = config_manager.get('discogs.token', '') + if _discogs_token: + sources.append(('discogs', _get_discogs_client(_discogs_token))) + except Exception as e: + logger.debug("discogs client init failed: %s", e) + # Hydrabase only when connected (dev-mode + active client). + try: + if hydrabase_client and hydrabase_client.is_connected(): + sources.append(('hydrabase', hydrabase_client)) + except Exception as e: + logger.debug("hydrabase client check failed: %s", e) + return sources + return _artists_quality.ArtistQualityDeps( - spotify_client=spotify_client, matching_engine=matching_engine, get_database=get_database, get_wishlist_service=_get_ws, get_current_profile_id=get_current_profile_id, get_quality_tier_from_extension=_get_quality_tier_from_extension, - get_metadata_fallback_client=_get_metadata_fallback_client, + get_metadata_search_sources=_resolve_search_sources, ) @@ -9717,13 +9757,7 @@ def get_track_tag_preview(track_id): # Include server type so frontend can offer server sync option active_server = config_manager.get_active_media_server() - server_connected = False - if active_server == 'plex': - server_connected = plex_client.is_connected() - elif active_server == 'jellyfin': - server_connected = jellyfin_client.is_connected() - elif active_server == 'navidrome': - server_connected = navidrome_client.is_connected() + server_connected = media_server_engine.is_connected() if media_server_engine else False return jsonify({ "success": True, @@ -9824,15 +9858,9 @@ def get_batch_tag_preview(): results.append(entry) - # Server type info + # Server type info — engine routes to active client. active_server = config_manager.get_active_media_server() - server_connected = False - if active_server == 'plex': - server_connected = plex_client.is_connected() - elif active_server == 'jellyfin': - server_connected = jellyfin_client.is_connected() - elif active_server == 'navidrome': - server_connected = navidrome_client.is_connected() + server_connected = media_server_engine.is_connected() if media_server_engine else False return jsonify({ "success": True, @@ -10983,7 +11011,7 @@ def _sync_tracks_to_server(track_rows, server_type): if track_data.get('year'): metadata['year'] = track_data['year'] if metadata: - success = plex_client.update_track_metadata(str(track_data['id']), metadata) + success = media_server_engine.client('plex').update_track_metadata(str(track_data['id']), metadata) if success: result['synced'] += 1 else: @@ -10998,7 +11026,7 @@ def _sync_tracks_to_server(track_rows, server_type): elif server_type == 'jellyfin': # Jellyfin: just trigger a library scan once after all file writes try: - success = jellyfin_client.trigger_library_scan() + success = media_server_engine.client('jellyfin').trigger_library_scan() if success: result['synced'] = len(track_rows) else: @@ -11023,14 +11051,17 @@ def _resolve_library_file_path(file_path): transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - # Also check the media server's music library path (handles Docker↔host path mismatch) + # Also check the media server's music library path (handles Docker↔host path mismatch). + # ``get_music_library_locations`` handles both single-library mode and + # all-libraries mode (unions location paths across every music section). library_dirs = set() try: - if plex_client and plex_client.server and plex_client.music_library: - for loc in plex_client.music_library.locations: + plex = media_server_engine.client('plex') + if plex and plex.server: + for loc in plex.get_music_library_locations(): library_dirs.add(loc) - except Exception: - pass + except Exception as e: + logger.debug("plex library locations lookup failed: %s", e) # Check user-configured music library paths (Settings > Library) try: @@ -11041,8 +11072,8 @@ def _resolve_library_file_path(file_path): resolved_p = docker_resolve_path(p.strip()) if resolved_p: library_dirs.add(resolved_p) - except Exception: - pass + except Exception as e: + logger.debug("library music paths read failed: %s", e) path_parts = file_path.replace('\\', '/').split('/') @@ -11533,8 +11564,8 @@ def library_delete_track(track_id): try: os.remove(sidecar) logger.info(f"Deleted sidecar file: {sidecar}") - except Exception: - pass + except Exception as e: + logger.debug("sidecar removal failed: %s", e) except Exception as e: logger.warning(f"Failed to delete file: {e}") file_error = str(e) @@ -11699,9 +11730,9 @@ def redownload_search_metadata(track_id): if thumb_url and not thumb_url.startswith('http'): _ab = '' _at = '' - if plex_client and plex_client.server: - _ab = getattr(plex_client.server, '_baseurl', '') or '' - _at = getattr(plex_client.server, '_token', '') or '' + if media_server_engine.client('plex') and media_server_engine.client('plex').server: + _ab = getattr(media_server_engine.client('plex').server, '_baseurl', '') or '' + _at = getattr(media_server_engine.client('plex').server, '_token', '') or '' if not _ab: _pc = config_manager.get_plex_config() _ab = (_pc.get('base_url', '') or '').rstrip('/') @@ -11723,21 +11754,16 @@ def redownload_search_metadata(track_id): 'thumb_url': thumb_url, } - # Search all available metadata sources in parallel - from concurrent.futures import ThreadPoolExecutor, as_completed - from difflib import SequenceMatcher + # Search all available metadata sources in parallel via the + # shared module — same code path the Enhance Quality flow uses + # so both endpoints have identical scoring + per-source query + # optimization + current-match flagging. + from core.metadata.multi_source_search import ( + TrackQuery, + search_all_sources, + ) - def _score_metadata_match(result): - """Score a metadata result against the current track.""" - title_sim = SequenceMatcher(None, track_title.lower(), (result.get('name') or '').lower()).ratio() - artist_sim = SequenceMatcher(None, artist_name.lower(), (result.get('artist') or '').lower()).ratio() - dur_diff = abs((row['duration'] or 0) - (result.get('duration_ms') or 0)) - dur_score = max(0, 1 - dur_diff / 30000) if row['duration'] else 0.5 - return round((title_sim * 0.5 + artist_sim * 0.35 + dur_score * 0.15), 3) - - metadata_results = {} sources_to_search = [] - if spotify_client and spotify_client.is_authenticated(): sources_to_search.append(('spotify', spotify_client)) try: @@ -11749,65 +11775,23 @@ def redownload_search_metadata(track_id): except Exception as e: logger.debug(f"Deezer client not available for redownload search: {e}") - # Build source-optimized queries - deezer_query = f'artist:"{artist_name}" track:"{clean_title}"' - - def _search_source(source_name, client): - try: - # Deezer works best with structured artist:/track: queries - search_q = deezer_query if source_name == 'deezer' else query - logger.info(f"[Redownload] Searching {source_name} for: {search_q}") - track_objs = client.search_tracks(search_q, limit=10) - # If no results, try plain query as fallback - if not track_objs and search_q != query: - track_objs = client.search_tracks(query, limit=10) - # Last resort: title only - if not track_objs and clean_title != query: - track_objs = client.search_tracks(clean_title, limit=10) - logger.info(f"[Redownload] {source_name} returned {len(track_objs)} results") - results = [] - for t in track_objs: - r = { - 'id': str(t.id), - 'name': t.name, - 'artist': ', '.join(t.artists) if t.artists else '', - 'album': t.album or '', - 'duration_ms': t.duration_ms or 0, - 'image_url': t.image_url or '', - 'is_current_match': False, - } - # Flag current match - if source_name == 'spotify' and row['spotify_track_id'] and str(t.id) == str(row['spotify_track_id']): - r['is_current_match'] = True - elif source_name == 'deezer' and row['deezer_id'] and str(t.id) == str(row['deezer_id']): - r['is_current_match'] = True - r['match_score'] = _score_metadata_match(r) - results.append(r) - results.sort(key=lambda x: (-int(x['is_current_match']), -x['match_score'])) - return source_name, results - except Exception as e: - logger.error(f"[Redownload] Metadata search failed for {source_name}: {e}", exc_info=True) - return source_name, [] - - with ThreadPoolExecutor(max_workers=3) as pool: - futures = {pool.submit(_search_source, name, client): name for name, client in sources_to_search} - for future in as_completed(futures): - source_name, results = future.result() - metadata_results[source_name] = results - - # Find best overall match - best_match = None - for source, results in metadata_results.items(): - if results: - top = results[0] - if not best_match or top['match_score'] > best_match['score']: - best_match = {'source': source, 'index': 0, 'score': top['match_score']} + track_query = TrackQuery( + title=track_title, + artist=artist_name, + album=row['album_title'] or '', + duration_ms=row['duration'] or 0, + spotify_track_id=row['spotify_track_id'], + deezer_id=row['deezer_id'], + ) + result = search_all_sources( + track_query, sources_to_search, clean_title=clean_title, + ) return jsonify({ "success": True, "current_track": current_track, - "metadata_results": metadata_results, - "best_match": best_match, + "metadata_results": result.metadata_results, + "best_match": result.best_match, }) except Exception as e: logger.error(f"Error in redownload metadata search: {e}", exc_info=True) @@ -11846,34 +11830,19 @@ def redownload_search_sources(track_id): candidates = [] database = get_database() - # Get all available download source clients + # Get all available download source clients via the orchestrator's + # generic accessor — replaces the old per-source if/hasattr chain + # that Cin called out as defeating the purpose of the registry refactor. download_clients = {} try: - orch = soulseek_client # The download orchestrator - if hasattr(orch, 'soulseek') and orch.soulseek: - if not (hasattr(orch.soulseek, 'is_configured') and not orch.soulseek.is_configured()): - download_clients['soulseek'] = orch.soulseek - if hasattr(orch, 'youtube') and orch.youtube: - if not (hasattr(orch.youtube, 'is_configured') and not orch.youtube.is_configured()): - download_clients['youtube'] = orch.youtube - if hasattr(orch, 'tidal') and orch.tidal: - if not (hasattr(orch.tidal, 'is_configured') and not orch.tidal.is_configured()): - download_clients['tidal'] = orch.tidal - if hasattr(orch, 'qobuz') and orch.qobuz: - if not (hasattr(orch.qobuz, 'is_configured') and not orch.qobuz.is_configured()): - download_clients['qobuz'] = orch.qobuz - if hasattr(orch, 'hifi') and orch.hifi: - if not (hasattr(orch.hifi, 'is_configured') and not orch.hifi.is_configured()): - download_clients['hifi'] = orch.hifi - if hasattr(orch, 'deezer_dl') and orch.deezer_dl: - if not (hasattr(orch.deezer_dl, 'is_configured') and not orch.deezer_dl.is_configured()): - download_clients['deezer_dl'] = orch.deezer_dl + if download_orchestrator and hasattr(download_orchestrator, 'configured_clients'): + download_clients = dict(download_orchestrator.configured_clients()) except Exception as e: logger.warning(f"[Redownload] Error getting download clients: {e}") if not download_clients: # Fallback: use orchestrator directly - download_clients = {'default': soulseek_client} + download_clients = {'default': download_orchestrator} logger.info(f"[Redownload] Streaming search across {len(download_clients)} sources: {list(download_clients.keys())}") @@ -11893,7 +11862,7 @@ def redownload_search_sources(track_id): quality = ext if ext in ('FLAC', 'MP3', 'OPUS', 'OGG', 'M4A', 'WAV') else candidate.quality or '' svc = source_name if source_name != 'default' else 'hybrid' uname = candidate.username - if uname in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): + if uname in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): svc = uname source_candidates.append({ 'username': uname, @@ -12007,12 +11976,12 @@ def sync_artist_library(artist_id): if server_source: media_client = None - if server_source == 'plex' and plex_client and plex_client.server: - media_client = plex_client - elif server_source == 'jellyfin' and jellyfin_client: - media_client = jellyfin_client - elif server_source == 'navidrome' and navidrome_client: - media_client = navidrome_client + if server_source == 'plex' and media_server_engine.client('plex') and media_server_engine.client('plex').server: + media_client = media_server_engine.client('plex') + elif server_source == 'jellyfin' and media_server_engine.client('jellyfin'): + media_client = media_server_engine.client('jellyfin') + elif server_source == 'navidrome' and media_server_engine.client('navidrome'): + media_client = media_server_engine.client('navidrome') if media_client: try: @@ -12156,8 +12125,8 @@ def library_delete_album(album_id): if os.path.exists(sidecar): try: os.remove(sidecar) - except Exception: - pass + except Exception as e: + logger.debug("sidecar removal failed: %s", e) except Exception as e: logger.warning(f"Failed to delete track file: {e}") files_failed += 1 @@ -12175,8 +12144,8 @@ def library_delete_album(album_id): if os.path.isdir(album_dir) and not os.listdir(album_dir): os.rmdir(album_dir) logger.info(f"Removed empty album directory: {album_dir}") - except Exception: - pass + except Exception as e: + logger.debug("empty album dir cleanup failed: %s", e) # Delete all tracks belonging to this album cursor.execute("DELETE FROM tracks WHERE album_id = ?", (album_id,)) @@ -12835,7 +12804,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar continue # Start download - download_id = run_async(soulseek_client.download(username, filename, size)) + download_id = run_async(download_orchestrator.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) @@ -12883,7 +12852,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar if not username or not filename: continue - download_id = run_async(soulseek_client.download(username, filename, size)) + download_id = run_async(download_orchestrator.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) @@ -12991,7 +12960,7 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): 'disc_number': corrected_meta.get('disc_number', 1) } - download_id = run_async(soulseek_client.download(username, filename, size)) + download_id = run_async(download_orchestrator.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) @@ -13058,7 +13027,7 @@ def start_matched_download(): if not username or not filename: return jsonify({"success": False, "error": "Missing username or filename"}), 400 - download_id = run_async(soulseek_client.download(username, filename, size)) + download_id = run_async(download_orchestrator.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) @@ -13118,7 +13087,7 @@ def start_matched_download(): download_payload['title'] = parsed_meta.get('title') or download_payload.get('title') download_payload['artist'] = parsed_meta.get('artist') or download_payload.get('artist') - download_id = run_async(soulseek_client.download(username, filename, size)) + download_id = run_async(download_orchestrator.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) @@ -13147,7 +13116,7 @@ def start_matched_download(): def _parse_filename_metadata(filename: str) -> dict: """ - A direct port of the metadata parsing logic from the GUI's soulseek_client.py. + A direct port of the metadata parsing logic from the GUI's download_orchestrator.py. This is the crucial missing step that cleans filenames BEFORE Spotify matching. """ return parse_filename_metadata(filename) @@ -13330,8 +13299,8 @@ def _sweep_empty_download_directories(): for hidden in entries: try: os.remove(os.path.join(dirpath, hidden)) - except Exception: - pass + except Exception as e: + logger.debug("hidden file cleanup failed: %s", e) os.rmdir(dirpath) removed += 1 except OSError: @@ -14143,8 +14112,8 @@ def _apply_path_template(template: str, context: dict) -> str: resolved = _get_itunes_client().resolve_primary_artist(itunes_artist_id) if resolved and resolved != album_artist_value: album_artist_value = resolved - except Exception: - pass + except Exception as e: + logger.debug("itunes primary artist resolve failed: %s", e) # $cdnum — smart CD label for multi-disc filenames. Produces "CD01" / # "CD02" etc. when the album has 2+ discs, empty string otherwise. # Empty output collapses gracefully via the trailing double-dash cleanup @@ -14313,7 +14282,18 @@ def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_in context, artist, album_info, - runtime=metadata_runtime or _build_metadata_enrichment_runtime(), + runtime=metadata_runtime or _build_metadata_enrichment_runtime( + mb_worker=mb_worker, + deezer_worker=deezer_worker, + audiodb_worker=audiodb_worker, + tidal_client=tidal_client, + qobuz_enrichment_worker=qobuz_enrichment_worker, + lastfm_worker=lastfm_worker, + genius_worker=genius_worker, + spotify_enrichment_worker=spotify_enrichment_worker, + itunes_enrichment_worker=itunes_enrichment_worker, + hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None, + ), ) @@ -14407,7 +14387,18 @@ def _post_process_matched_download_with_verification(context_key, context, file_ task_id, batch_id, _build_import_pipeline_runtime(), - _build_metadata_enrichment_runtime(), + _build_metadata_enrichment_runtime( + mb_worker=mb_worker, + deezer_worker=deezer_worker, + audiodb_worker=audiodb_worker, + tidal_client=tidal_client, + qobuz_enrichment_worker=qobuz_enrichment_worker, + lastfm_worker=lastfm_worker, + genius_worker=genius_worker, + spotify_enrichment_worker=spotify_enrichment_worker, + itunes_enrichment_worker=itunes_enrichment_worker, + hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None, + ), ) @@ -14520,7 +14511,18 @@ def _post_process_matched_download(context_key, context, file_path): context, file_path, _build_import_pipeline_runtime(), - metadata_runtime=_build_metadata_enrichment_runtime(), + metadata_runtime=_build_metadata_enrichment_runtime( + mb_worker=mb_worker, + deezer_worker=deezer_worker, + audiodb_worker=audiodb_worker, + tidal_client=tidal_client, + qobuz_enrichment_worker=qobuz_enrichment_worker, + lastfm_worker=lastfm_worker, + genius_worker=genius_worker, + spotify_enrichment_worker=spotify_enrichment_worker, + itunes_enrichment_worker=itunes_enrichment_worker, + hifi_client=download_orchestrator.client("hifi") if download_orchestrator else None, + ), ) # Track stale transfer keys (completed in slskd but no context — e.g., from before app restart) @@ -14591,8 +14593,8 @@ def _get_current_commit_sha(): result = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True, cwd=os.path.dirname(__file__) or '.') if result.returncode == 0: return result.stdout.strip() - except Exception: - pass + except Exception as e: + logger.debug("git rev-parse failed: %s", e) return None _current_commit_sha = _get_current_commit_sha() @@ -14909,8 +14911,8 @@ def _db_update_finished_callback(total_artists, total_albums, total_tracks, succ 'total_albums': str(total_albums), 'total_tracks': str(total_tracks), }) - except Exception: - pass + except Exception as e: + logger.debug("library_updated automation emit failed: %s", e) # Invalidate sync match cache (track IDs may have changed) try: @@ -14918,8 +14920,8 @@ def _db_update_finished_callback(total_artists, total_albums, total_tracks, succ cleared = inv_db.invalidate_sync_match_cache() if cleared: logger.info(f"Cleared {cleared} sync match cache entries after database update") - except Exception: - pass + except Exception as e: + logger.debug("sync match cache invalidation failed: %s", e) # WISHLIST CLEANUP: Automatically clean up wishlist after database update try: @@ -15050,8 +15052,8 @@ def _run_soulsync_full_refresh(): INSERT OR IGNORE INTO artists (id, name, server_source, created_at, updated_at) VALUES (?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, (artist_id, artist_name)) - except Exception: - pass + except Exception as e: + logger.debug("soulsync artist insert failed: %s", e) for album_name, tracks in albums.items(): album_key = f"{artist_name.lower()}::{album_name.lower()}" @@ -15070,8 +15072,8 @@ def _run_soulsync_full_refresh(): INSERT OR IGNORE INTO albums (id, artist_id, title, year, track_count, server_source, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, (album_id, artist_id, album_name, year, len(tracks))) - except Exception: - pass + except Exception as e: + logger.debug("soulsync album insert failed: %s", e) # Insert tracks for file_path, tags in tracks: @@ -15262,11 +15264,11 @@ def _run_db_update_task(full_refresh, server_type): media_client = None if server_type == "plex": - media_client = plex_client + media_client = media_server_engine.client('plex') elif server_type == "jellyfin": - media_client = jellyfin_client + media_client = media_server_engine.client('jellyfin') elif server_type == "navidrome": - media_client = navidrome_client + media_client = media_server_engine.client('navidrome') if not media_client: _db_update_error_callback(f"Media client for '{server_type}' not available.") @@ -15308,11 +15310,11 @@ def _run_deep_scan_task(server_type): media_client = None if server_type == "plex": - media_client = plex_client + media_client = media_server_engine.client('plex') elif server_type == "jellyfin": - media_client = jellyfin_client + media_client = media_server_engine.client('jellyfin') elif server_type == "navidrome": - media_client = navidrome_client + media_client = media_server_engine.client('navidrome') elif server_type == "soulsync": # SoulSync standalone deep scan: find untracked files → move to Staging, # remove stale DB records where files no longer exist on disk @@ -15861,8 +15863,8 @@ def backup_database_endpoint(): try: with open(meta_path, 'w') as mf: json.dump({"version": SOULSYNC_VERSION, "created": timestamp}, mf) - except Exception: - pass # Non-critical — backup still works without metadata + except Exception as e: + logger.debug("backup meta sidecar write: %s", e) # Rolling cleanup existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime) # Filter out .meta.json files from the backup list @@ -15874,8 +15876,8 @@ def backup_database_endpoint(): # Also remove sidecar if present if os.path.exists(removed + '.meta.json'): os.remove(removed + '.meta.json') - except Exception: - pass + except Exception as e: + logger.debug("rolling backup cleanup failed: %s", e) return jsonify({"success": True, "backup_path": backup_path, "size_mb": size_mb, "version": SOULSYNC_VERSION}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @@ -15909,8 +15911,8 @@ def list_backups_endpoint(): with open(meta_path, 'r') as mf: meta = json.load(mf) entry['version'] = meta.get('version') - except Exception: - pass + except Exception as e: + logger.debug("backup metadata read failed: %s", e) backups.append(entry) db_size_mb = round(os.path.getsize(db_path) / (1024 * 1024), 2) if os.path.exists(db_path) else 0 return jsonify({ @@ -15938,8 +15940,8 @@ def delete_backup_endpoint(filename): if os.path.exists(meta_path): try: os.remove(meta_path) - except Exception: - pass + except Exception as e: + logger.debug("backup sidecar removal failed: %s", e) return jsonify({"success": True, "deleted": filename}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @@ -15965,8 +15967,8 @@ def restore_backup_endpoint(filename): with open(meta_path, 'r') as mf: meta = json.load(mf) backup_version = meta.get('version') - except Exception: - pass + except Exception as e: + logger.debug("backup version metadata read failed: %s", e) version_warning = None # Compare base versions only (strip +commit suffix) to avoid false mismatches @@ -15998,8 +16000,8 @@ def restore_backup_endpoint(filename): try: with open(safety_path + '.meta.json', 'w') as mf: json.dump({"version": SOULSYNC_VERSION, "created": safety_ts}, mf) - except Exception: - pass + except Exception as e: + logger.debug("safety backup metadata write failed: %s", e) # Restore using SQLite backup API (handles concurrent access safely) from database.music_database import close_database, get_database @@ -16963,7 +16965,7 @@ def _build_master_deps(): return _downloads_master.MasterDeps( config_manager=config_manager, - soulseek_client=soulseek_client, + download_orchestrator=download_orchestrator, run_async=run_async, mb_worker=mb_worker, mb_release_cache=mb_release_cache, @@ -17001,7 +17003,7 @@ def _build_post_processing_deps(): """Build the PostProcessDeps bundle from web_server.py globals on each call.""" return _downloads_post_processing.PostProcessDeps( config_manager=config_manager, - soulseek_client=soulseek_client, + download_orchestrator=download_orchestrator, run_async=run_async, docker_resolve_path=docker_resolve_path, extract_filename=extract_filename, @@ -17028,7 +17030,7 @@ from core.downloads import task_worker as _downloads_task_worker def _build_task_worker_deps(): """Build TaskWorkerDeps bundle from web_server.py globals on each call.""" return _downloads_task_worker.TaskWorkerDeps( - soulseek_client=soulseek_client, + download_orchestrator=download_orchestrator, matching_engine=matching_engine, run_async=run_async, try_source_reuse=_try_source_reuse, @@ -17054,7 +17056,7 @@ from core.downloads import candidates as _downloads_candidates def _build_candidates_deps(): """Build the CandidatesDeps bundle from web_server.py globals on each call.""" return _downloads_candidates.CandidatesDeps( - soulseek_client=soulseek_client, + download_orchestrator=download_orchestrator, spotify_client=spotify_client, run_async=run_async, get_database=get_database, @@ -17163,7 +17165,7 @@ def _try_source_reuse(task_id, batch_id, track): if not source_tracks or not last_source: _sr.info("Skipped — no source_tracks or no last_source") return False - if last_source.get('username') in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): + if last_source.get('username') in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): _sr.info(f"Skipped — {last_source.get('username')} source (no folder-based reuse)") return False @@ -17219,7 +17221,7 @@ def _try_source_reuse(task_id, batch_id, track): # Sort by confidence, filter by quality preference candidates.sort(key=lambda c: c.confidence, reverse=True) _sr.info(f"Found {len(candidates)} candidates above 0.70, best={candidates[0].confidence:.3f} ({candidates[0].filename})") - slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client + slsk = download_orchestrator.client("soulseek") if hasattr(download_orchestrator, 'client') else download_orchestrator filtered = slsk.filter_results_by_quality_preference(candidates) if not filtered: _sr.info(f"Quality filter rejected all candidates for task {task_id}") @@ -17265,7 +17267,7 @@ def _store_batch_source(batch_id, username, filename): """Browse the successful download's folder and store results on the batch for reuse.""" _sr = source_reuse_logger _sr.info(f"_store_batch_source called: batch={batch_id}, user={username}, file={filename}") - if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): + if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): _sr.info(f"Skipped — no batch_id or streaming source ({username})") return @@ -17298,8 +17300,8 @@ def _store_batch_source(batch_id, username, filename): return try: - # Access SoulseekClient directly (soulseek_client is DownloadOrchestrator) - slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client + # Access SoulseekClient directly (download_orchestrator is DownloadOrchestrator) + slsk = download_orchestrator.client("soulseek") if hasattr(download_orchestrator, 'client') else download_orchestrator _sr.info(f"Browsing {username}:{folder_path}...") files = run_async(slsk.browse_user_directory(username, folder_path)) if not files: @@ -17630,7 +17632,7 @@ def cancel_download_task(): if download_id and username: try: # This is an async call, so we run it and wait - run_async(soulseek_client.cancel_download(download_id, username, remove=True)) + run_async(download_orchestrator.cancel_download(download_id, username, remove=True)) logger.warning(f"Successfully cancelled Soulseek download {download_id} for task {task_id}") except Exception as e: logger.error(f"Failed to cancel download on slskd, but worker already moved on: {e}") @@ -17875,14 +17877,14 @@ def cancel_task_v2(): # username: youtube/tidal/qobuz/hifi/deezer_dl/lidarr go to # their streaming clients, anything else goes to Soulseek. # - # Replaces an older block that assumed soulseek_client was a + # Replaces an older block that assumed download_orchestrator was a # raw SoulseekClient and accessed .base_url / ._make_request # directly — crashed with AttributeError on the orchestrator # and silently left streaming downloads running in background. try: logger.info(f"[Atomic Cancel] Dispatching cancel to orchestrator: username={username} download_id={download_id}") cancel_success = run_async( - soulseek_client.cancel_download(download_id, username, remove=True) + download_orchestrator.cancel_download(download_id, username, remove=True) ) if cancel_success: logger.info(f"[Atomic Cancel] Orchestrator cancelled download: {download_id}") @@ -18185,10 +18187,10 @@ def get_server_playlists(): return jsonify({"success": False, "error": "No media server configured"}), 400 playlists_data = [] - if active_server == 'plex' and plex_client and plex_client.is_connected(): + if active_server == 'plex' and media_server_engine.client('plex') and media_server_engine.client('plex').is_connected(): # Use raw Plex API to get playlist metadata without fetching all tracks try: - raw_playlists = plex_client.server.playlists() + raw_playlists = media_server_engine.client('plex').server.playlists() logger.info(f"[ServerPlaylists] Plex returned {len(raw_playlists)} total playlists") for playlist in raw_playlists: if getattr(playlist, 'playlistType', None) == 'audio': @@ -18201,22 +18203,22 @@ def get_server_playlists(): except Exception as e: logger.error(f"[ServerPlaylists] Error fetching Plex playlists: {e}", exc_info=True) return jsonify({"success": False, "error": f"Plex error: {str(e)}"}), 500 - elif active_server == 'jellyfin' and jellyfin_client and jellyfin_client.is_connected(): - for pl in jellyfin_client.get_all_playlists(): + elif active_server == 'jellyfin' and media_server_engine.client('jellyfin') and media_server_engine.client('jellyfin').is_connected(): + for pl in media_server_engine.client('jellyfin').get_all_playlists(): playlists_data.append({ 'id': pl.id, 'name': pl.title, 'track_count': pl.leaf_count, }) - elif active_server == 'navidrome' and navidrome_client and navidrome_client.is_connected(): - for pl in navidrome_client.get_all_playlists(): + elif active_server == 'navidrome' and media_server_engine.client('navidrome') and media_server_engine.client('navidrome').is_connected(): + for pl in media_server_engine.client('navidrome').get_all_playlists(): playlists_data.append({ 'id': pl.id, 'name': pl.title, 'track_count': pl.leaf_count, }) else: - logger.warning(f"[ServerPlaylists] Server '{active_server}' not connected. plex_client={plex_client is not None}, jellyfin_client={jellyfin_client is not None}, navidrome_client={navidrome_client is not None}") + logger.warning(f"[ServerPlaylists] Server '{active_server}' not connected. plex_client={media_server_engine.client('plex') is not None}, jellyfin_client={media_server_engine.client('jellyfin') is not None}, navidrome_client={media_server_engine.client('navidrome') is not None}") return jsonify({"success": False, "error": f"{active_server} not connected"}), 400 return jsonify({"success": True, "server_type": active_server, "playlists": playlists_data}) @@ -18234,24 +18236,30 @@ def get_server_playlist_tracks(playlist_id): # Get tracks from server server_tracks = [] - if active_server == 'plex' and plex_client: + if active_server == 'plex' and media_server_engine.client('plex'): try: # Try by ID first, fall back to name lookup (ID changes when playlist is recreated) raw_playlist = None try: - raw_playlist = plex_client.server.fetchItem(int(playlist_id)) - except Exception: - pass + raw_playlist = media_server_engine.client('plex').server.fetchItem(int(playlist_id)) + except Exception as e: + logger.debug("plex playlist fetchItem failed: %s", e) if not raw_playlist and playlist_name: try: - raw_playlist = plex_client.server.playlist(playlist_name) - except Exception: - pass + raw_playlist = media_server_engine.client('plex').server.playlist(playlist_name) + except Exception as e: + logger.debug("plex playlist by-name lookup failed: %s", e) + if not raw_playlist: + logger.warning( + f"[ServerPlaylistTracks] Plex playlist not found by " + f"id={playlist_id} or name='{playlist_name}' — " + f"compare view will show every source track as Find & Add" + ) if raw_playlist: if not playlist_name: playlist_name = raw_playlist.title - plex_base = getattr(plex_client.server, '_baseurl', '') or '' - plex_token = getattr(plex_client.server, '_token', '') or '' + plex_base = getattr(media_server_engine.client('plex').server, '_baseurl', '') or '' + plex_token = getattr(media_server_engine.client('plex').server, '_token', '') or '' if not plex_base: # Fallback: get from config _pc = config_manager.get_plex_config() @@ -18276,9 +18284,9 @@ def get_server_playlist_tracks(playlist_id): }) except Exception as e: logger.error(f"[ServerPlaylistTracks] Plex error: {e}", exc_info=True) - elif active_server == 'jellyfin' and jellyfin_client: - tracks = jellyfin_client.get_playlist_tracks(playlist_id) - jf_base = jellyfin_client.base_url or '' + elif active_server == 'jellyfin' and media_server_engine.client('jellyfin'): + tracks = media_server_engine.client('jellyfin').get_playlist_tracks(playlist_id) + jf_base = media_server_engine.client('jellyfin').base_url or '' for t in (tracks or []): raw = t._data if hasattr(t, '_data') else {} artists = raw.get('Artists', []) @@ -18293,8 +18301,8 @@ def get_server_playlist_tracks(playlist_id): 'duration': t.duration, 'thumb': thumb, }) - elif active_server == 'navidrome' and navidrome_client: - tracks = navidrome_client.get_playlist_tracks(playlist_id) + elif active_server == 'navidrome' and media_server_engine.client('navidrome'): + tracks = media_server_engine.client('navidrome').get_playlist_tracks(playlist_id) for t in (tracks or []): raw = t._data if hasattr(t, '_data') else {} # Navidrome cover art via Subsonic API @@ -18319,9 +18327,9 @@ def get_server_playlist_tracks(playlist_id): # Build server art URL prefix for resolving relative thumb paths _art_prefix = '' _art_suffix = '' - if active_server == 'plex' and plex_client and plex_client.server: - _ab = getattr(plex_client.server, '_baseurl', '') or '' - _at = getattr(plex_client.server, '_token', '') or '' + if active_server == 'plex' and media_server_engine.client('plex') and media_server_engine.client('plex').server: + _ab = getattr(media_server_engine.client('plex').server, '_baseurl', '') or '' + _at = getattr(media_server_engine.client('plex').server, '_token', '') or '' if not _ab: _pc = config_manager.get_plex_config() _ab = (_pc.get('base_url', '') or '').rstrip('/') @@ -18504,11 +18512,22 @@ def server_playlist_replace_track(playlist_id): active_server = config_manager.get_active_media_server() - if active_server == 'plex' and plex_client: - # Use raw Plex API - fetch playlist directly + if active_server == 'plex' and media_server_engine.client('plex'): + # ID-first, name-fallback (Plex deletes + recreates on edit + # so the cached rating key can be stale). + plex_server = media_server_engine.client('plex').server + raw_playlist = None try: - raw_playlist = plex_client.server.playlist(playlist_name) - except Exception: + raw_playlist = plex_server.fetchItem(int(playlist_id)) + except Exception as e: + logger.debug("plex playlist fetchItem failed: %s", e) + if not raw_playlist and playlist_name: + try: + raw_playlist = plex_server.playlist(playlist_name) + except Exception as e: + logger.debug("plex playlist by-name lookup failed: %s", e) + if not raw_playlist: + logger.warning(f"[ServerPlaylist] replace-track: playlist not found by id={playlist_id} or name='{playlist_name}'") return jsonify({"success": False, "error": "Playlist not found on server"}), 404 # Build new track list with replacement @@ -18516,7 +18535,7 @@ def server_playlist_replace_track(playlist_id): replaced = False for item in raw_playlist.items(): if str(item.ratingKey) == str(old_track_id) and not replaced: - new_item = plex_client.server.fetchItem(int(new_track_id)) + new_item = media_server_engine.client('plex').server.fetchItem(int(new_track_id)) if new_item: new_tracks.append(new_item) replaced = True @@ -18529,13 +18548,13 @@ def server_playlist_replace_track(playlist_id): # Delete old and recreate directly (avoid update_playlist's backup logic) raw_playlist.delete() from plexapi.playlist import Playlist - new_pl = Playlist.create(plex_client.server, playlist_name, items=new_tracks) + new_pl = Playlist.create(media_server_engine.client('plex').server, playlist_name, items=new_tracks) return jsonify({"success": True, "message": "Track replaced", "new_playlist_id": str(new_pl.ratingKey)}) else: return jsonify({"success": False, "error": "Old track not found in playlist"}), 404 - elif active_server == 'jellyfin' and jellyfin_client: - current_tracks = jellyfin_client.get_playlist_tracks(playlist_id) + elif active_server == 'jellyfin' and media_server_engine.client('jellyfin'): + current_tracks = media_server_engine.client('jellyfin').get_playlist_tracks(playlist_id) new_track_ids = [] replaced = False for t in (current_tracks or []): @@ -18548,12 +18567,12 @@ def server_playlist_replace_track(playlist_id): if replaced: new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_track_ids] - jellyfin_client.update_playlist(playlist_name, new_track_objs) + media_server_engine.client('jellyfin').update_playlist(playlist_name, new_track_objs) return jsonify({"success": True, "message": "Track replaced"}) return jsonify({"success": False, "error": "Old track not found"}), 404 - elif active_server == 'navidrome' and navidrome_client: - current_tracks = navidrome_client.get_playlist_tracks(playlist_id) + elif active_server == 'navidrome' and media_server_engine.client('navidrome'): + current_tracks = media_server_engine.client('navidrome').get_playlist_tracks(playlist_id) new_track_ids = [] replaced = False for t in (current_tracks or []): @@ -18566,7 +18585,7 @@ def server_playlist_replace_track(playlist_id): if replaced: new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_track_ids] - navidrome_client.create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) + media_server_engine.client('navidrome').create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) return jsonify({"success": True, "message": "Track replaced"}) return jsonify({"success": False, "error": "Old track not found"}), 404 @@ -18592,12 +18611,26 @@ def server_playlist_add_track(playlist_id): active_server = config_manager.get_active_media_server() - if active_server == 'plex' and plex_client: + if active_server == 'plex' and media_server_engine.client('plex'): + # ID-first, name-fallback — Plex deletes + recreates playlists + # on edit so the rating key the frontend cached can be stale. + # The GET tracks endpoint uses the same lookup chain. + plex_server = media_server_engine.client('plex').server + raw_playlist = None try: - raw_playlist = plex_client.server.playlist(playlist_name) - except Exception: + raw_playlist = plex_server.fetchItem(int(playlist_id)) + except Exception as e: + logger.debug("plex playlist fetchItem failed: %s", e) + if not raw_playlist and playlist_name: + try: + raw_playlist = plex_server.playlist(playlist_name) + except Exception as e: + logger.debug("plex playlist by-name lookup failed: %s", e) + if not raw_playlist: + logger.warning(f"[ServerPlaylist] add-track: playlist not found by id={playlist_id} or name='{playlist_name}'") return jsonify({"success": False, "error": "Playlist not found"}), 404 - new_item = plex_client.server.fetchItem(int(track_id)) + + new_item = plex_server.fetchItem(int(track_id)) if not new_item: return jsonify({"success": False, "error": "Track not found on server"}), 404 @@ -18624,22 +18657,22 @@ def server_playlist_add_track(playlist_id): logger.info(f"[ServerPlaylist] Added track to playlist, playlist ID: {new_id}") return jsonify({"success": True, "message": "Track added", "new_playlist_id": new_id}) - elif active_server == 'jellyfin' and jellyfin_client: - current_tracks = jellyfin_client.get_playlist_tracks(playlist_id) or [] + elif active_server == 'jellyfin' and media_server_engine.client('jellyfin'): + current_tracks = media_server_engine.client('jellyfin').get_playlist_tracks(playlist_id) or [] track_ids = [str(t.ratingKey) for t in current_tracks] pos = max(0, min(int(position), len(track_ids))) if position is not None else len(track_ids) track_ids.insert(pos, track_id) new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in track_ids] - jellyfin_client.update_playlist(playlist_name, new_track_objs) + media_server_engine.client('jellyfin').update_playlist(playlist_name, new_track_objs) return jsonify({"success": True, "message": "Track added"}) - elif active_server == 'navidrome' and navidrome_client: - current_tracks = navidrome_client.get_playlist_tracks(playlist_id) or [] + elif active_server == 'navidrome' and media_server_engine.client('navidrome'): + current_tracks = media_server_engine.client('navidrome').get_playlist_tracks(playlist_id) or [] track_ids = [str(t.ratingKey) for t in current_tracks] pos = max(0, min(int(position), len(track_ids))) if position is not None else len(track_ids) track_ids.insert(pos, track_id) new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in track_ids] - navidrome_client.create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) + media_server_engine.client('navidrome').create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) return jsonify({"success": True, "message": "Track added"}) return jsonify({"success": False, "error": f"Unsupported server: {active_server}"}), 400 @@ -18663,10 +18696,22 @@ def server_playlist_remove_track(playlist_id): active_server = config_manager.get_active_media_server() - if active_server == 'plex' and plex_client: + if active_server == 'plex' and media_server_engine.client('plex'): + # ID-first, name-fallback (Plex deletes + recreates on edit + # so the cached rating key can be stale). + plex_server = media_server_engine.client('plex').server + raw_playlist = None try: - raw_playlist = plex_client.server.playlist(playlist_name) - except Exception: + raw_playlist = plex_server.fetchItem(int(playlist_id)) + except Exception as e: + logger.debug("plex playlist fetchItem failed: %s", e) + if not raw_playlist and playlist_name: + try: + raw_playlist = plex_server.playlist(playlist_name) + except Exception as e: + logger.debug("plex playlist by-name lookup failed: %s", e) + if not raw_playlist: + logger.warning(f"[ServerPlaylist] remove-track: playlist not found by id={playlist_id} or name='{playlist_name}'") return jsonify({"success": False, "error": "Playlist not found"}), 404 # Rebuild without the target track @@ -18677,26 +18722,26 @@ def server_playlist_remove_track(playlist_id): raw_playlist.delete() if new_items: from plexapi.playlist import Playlist - new_pl = Playlist.create(plex_client.server, playlist_name, items=new_items) + new_pl = Playlist.create(media_server_engine.client('plex').server, playlist_name, items=new_items) return jsonify({"success": True, "message": "Track removed", "new_playlist_id": str(new_pl.ratingKey)}) return jsonify({"success": True, "message": "Track removed (playlist now empty)"}) - elif active_server == 'jellyfin' and jellyfin_client: - current_tracks = jellyfin_client.get_playlist_tracks(playlist_id) or [] + elif active_server == 'jellyfin' and media_server_engine.client('jellyfin'): + current_tracks = media_server_engine.client('jellyfin').get_playlist_tracks(playlist_id) or [] new_ids = [str(t.ratingKey) for t in current_tracks if str(t.ratingKey) != str(remove_track_id)] if len(new_ids) == len(current_tracks): return jsonify({"success": False, "error": "Track not found in playlist"}), 404 new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_ids] - jellyfin_client.update_playlist(playlist_name, new_track_objs) + media_server_engine.client('jellyfin').update_playlist(playlist_name, new_track_objs) return jsonify({"success": True, "message": "Track removed"}) - elif active_server == 'navidrome' and navidrome_client: - current_tracks = navidrome_client.get_playlist_tracks(playlist_id) or [] + elif active_server == 'navidrome' and media_server_engine.client('navidrome'): + current_tracks = media_server_engine.client('navidrome').get_playlist_tracks(playlist_id) or [] new_ids = [str(t.ratingKey) for t in current_tracks if str(t.ratingKey) != str(remove_track_id)] if len(new_ids) == len(current_tracks): return jsonify({"success": False, "error": "Track not found in playlist"}), 404 new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_ids] - navidrome_client.create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) + media_server_engine.client('navidrome').create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) return jsonify({"success": True, "message": "Track removed"}) return jsonify({"success": False, "error": f"Unsupported server: {active_server}"}), 400 @@ -18720,9 +18765,9 @@ def library_search_tracks(): # Build thumb URL resolver for this server _art_prefix = '' _art_suffix = '' - if active_server == 'plex' and plex_client and plex_client.server: - _ab = getattr(plex_client.server, '_baseurl', '') or '' - _at = getattr(plex_client.server, '_token', '') or '' + if active_server == 'plex' and media_server_engine.client('plex') and media_server_engine.client('plex').server: + _ab = getattr(media_server_engine.client('plex').server, '_baseurl', '') or '' + _at = getattr(media_server_engine.client('plex').server, '_token', '') or '' if not _ab: _pc = config_manager.get_plex_config() _ab = (_pc.get('base_url', '') or '').rstrip('/') @@ -19776,6 +19821,81 @@ def get_discover_album(source, album_id): 'source': fallback_source, }) + elif source == 'discogs': + # Discogs release detail. release_id comes from the Your + # Albums Discogs source. Tracklist needs normalizing — + # Discogs uses {position, title, duration} (duration as + # string like "3:45") so map to the standard + # {name, track_number, duration_ms, artists} shape the + # download modal expects. + from core.discogs_client import DiscogsClient + try: + rel_id = int(album_id) + except (TypeError, ValueError): + return jsonify({"error": "Invalid Discogs release id"}), 400 + + release = DiscogsClient().get_release(rel_id) + if not release: + return jsonify({"error": "Discogs release not found"}), 404 + + import re as _re + _disambig_re = _re.compile(r'\s*\(\d+\)$') + artists_raw = release.get('artists') or [] + artist_names = [] + for a in artists_raw: + name = (a.get('name') or '').strip() if isinstance(a, dict) else str(a) + # Strip Discogs disambiguation suffix "(N)" + name = _disambig_re.sub('', name) + if name: + artist_names.append({'name': name}) + + tracks_out = [] + for idx, t in enumerate(release.get('tracklist', []) or [], start=1): + if not isinstance(t, dict): + continue + title = (t.get('title') or '').strip() + if not title: + continue + # Discogs duration: "3:45" or "1:23:45". Convert to ms. + dur_ms = 0 + dur_str = (t.get('duration') or '').strip() + if dur_str: + try: + parts = [int(p) for p in dur_str.split(':')] + if len(parts) == 2: + dur_ms = (parts[0] * 60 + parts[1]) * 1000 + elif len(parts) == 3: + dur_ms = (parts[0] * 3600 + parts[1] * 60 + parts[2]) * 1000 + except (ValueError, TypeError): + dur_ms = 0 + tracks_out.append({ + 'id': f"discogs_{rel_id}_{idx}", + 'name': title, + 'track_number': idx, + 'duration_ms': dur_ms, + 'artists': artist_names, + }) + + images = release.get('images') or [] + cover_url = '' + if images and isinstance(images[0], dict): + cover_url = images[0].get('uri') or images[0].get('uri150') or '' + + year = release.get('year') + release_date = str(year) if year and int(year) > 0 else '' + + return jsonify({ + 'id': str(rel_id), + 'name': release.get('title', ''), + 'artists': artist_names, + 'release_date': release_date, + 'total_tracks': len(tracks_out), + 'album_type': 'album', + 'images': [{'url': cover_url}] if cover_url else [], + 'tracks': tracks_out, + 'source': 'discogs', + }) + else: return jsonify({"error": f"Unknown source: {source}"}), 400 @@ -19792,7 +19912,7 @@ def get_discover_album(source, album_id): def hifi_status(): """Check if HiFi API instances are reachable.""" try: - hifi = soulseek_client.hifi + hifi = download_orchestrator.client("hifi") available = hifi.is_available() version = hifi.get_version() if available else None return jsonify({ @@ -19804,12 +19924,46 @@ def hifi_status(): return jsonify({"available": False, "error": str(e)}) +@app.route('/api/soundcloud/status', methods=['GET']) +def soundcloud_status(): + """Report SoundCloud client availability + a quick reachability probe. + + SoundCloud anonymous mode needs no credentials, so "configured" is + really "yt-dlp is installed and SoundCloud responds to a search." + The check fans out a real (cheap) yt-dlp call so the settings page's + Test Connection button gives a meaningful pass/fail signal instead + of just verifying the import succeeded. + """ + try: + sc = download_orchestrator.client("soundcloud") if download_orchestrator and hasattr(download_orchestrator, 'client') else None + if not sc: + return jsonify({ + "available": False, + "configured": False, + "error": "SoundCloud client not initialized — check yt-dlp install", + }) + if not sc.is_available(): + return jsonify({ + "available": False, + "configured": False, + "error": "yt-dlp not installed", + }) + reachable = run_async(sc.check_connection()) + return jsonify({ + "available": True, + "configured": True, + "reachable": bool(reachable), + }) + except Exception as exc: + return jsonify({"available": False, "configured": False, "error": str(exc)}) + + @app.route('/api/hifi/instances', methods=['GET']) def hifi_instances(): """Check availability of all HiFi API instances.""" import requests as req try: - hifi = soulseek_client.hifi + hifi = download_orchestrator.client("hifi") instances = list(hifi._instances) results = [] for url in instances: @@ -19864,9 +20018,9 @@ def hifi_add_instance(): added = db.add_hifi_instance(url, priority) if not added: return jsonify({'success': False, 'error': 'Instance already exists'}), 400 - # Reload the client - if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi: - soulseek_client.hifi.reload_instances() + # Reload the HiFi client + if download_orchestrator: + download_orchestrator.reload_instances('hifi') return jsonify({'success': True, 'url': url}) except Exception as e: logger.error(f"Error adding HiFi instance: {e}") @@ -19886,9 +20040,9 @@ def hifi_remove_instance(): removed = db.remove_hifi_instance(url) if not removed: return jsonify({'success': False, 'error': 'Instance not found'}), 404 - # Reload the client - if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi: - soulseek_client.hifi.reload_instances() + # Reload the HiFi client + if download_orchestrator: + download_orchestrator.reload_instances('hifi') return jsonify({'success': True, 'url': url}) except Exception as e: logger.error(f"Error removing HiFi instance: {e}") @@ -19908,8 +20062,8 @@ def hifi_toggle_instance(): from database.music_database import get_database db = get_database() db.toggle_hifi_instance(url, enabled) - if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi: - soulseek_client.hifi.reload_instances() + if download_orchestrator: + download_orchestrator.reload_instances('hifi') return jsonify({'success': True}) except Exception as e: logger.error(f"Error toggling HiFi instance: {e}") @@ -19929,9 +20083,9 @@ def hifi_reorder_instances(): db = get_database() if not db.reorder_hifi_instances(urls): return jsonify({'success': False, 'error': 'One or more URLs not found'}), 400 - # Reload the client - if soulseek_client and hasattr(soulseek_client, 'hifi') and soulseek_client.hifi: - soulseek_client.hifi.reload_instances() + # Reload the HiFi client + if download_orchestrator: + download_orchestrator.reload_instances('hifi') return jsonify({'success': True}) except Exception as e: logger.error(f"Error reordering HiFi instances: {e}") @@ -20083,11 +20237,12 @@ def deezer_download_test_download(): def _get_tidal_download_client(): """Get Tidal download client from the orchestrator, with helpful error if unavailable.""" - if not soulseek_client: + if not download_orchestrator: raise RuntimeError("Download orchestrator not initialized — check startup logs for errors") - if not hasattr(soulseek_client, 'tidal') or not soulseek_client.tidal: + tidal = download_orchestrator.client("tidal") if hasattr(download_orchestrator, 'client') else None + if not tidal: raise RuntimeError("Tidal download client not available — ensure tidalapi is installed") - return soulseek_client.tidal + return tidal @app.route('/api/tidal/download/auth/start', methods=['POST']) def tidal_download_auth_start(): @@ -20129,6 +20284,22 @@ def tidal_download_auth_status(): # QOBUZ AUTH ENDPOINTS # =================================================================== +def _sync_qobuz_credentials_to_worker(): + """Push the just-saved Qobuz session into the enrichment worker's + QobuzClient. Two separate client instances run side by side (one for + the auth endpoints, one for the worker thread); without this sync the + worker's instance never sees the new token until the next process + restart, which is what made the dashboard indicator stay yellow and + the connection test return ``Qobuz not authenticated`` after a + successful Connect.""" + try: + worker = qobuz_enrichment_worker if 'qobuz_enrichment_worker' in globals() else None + if worker and getattr(worker, 'client', None): + worker.client.reload_credentials() + except Exception as e: + logger.debug(f"Could not sync Qobuz credentials to enrichment worker: {e}") + + @app.route('/api/qobuz/auth/login', methods=['POST']) def qobuz_auth_login(): """Login to Qobuz with email/password.""" @@ -20140,10 +20311,11 @@ def qobuz_auth_login(): if not email or not password: return jsonify({"success": False, "error": "Email and password required"}), 400 - qobuz = soulseek_client.qobuz + qobuz = download_orchestrator.client("qobuz") result = qobuz.login(email, password) if result['status'] == 'success': + _sync_qobuz_credentials_to_worker() return jsonify({"success": True, **result}) else: return jsonify({"success": False, "error": result.get('message', 'Login failed')}), 400 @@ -20162,10 +20334,11 @@ def qobuz_auth_token(): if not token: return jsonify({"success": False, "error": "Auth token required"}), 400 - qobuz = soulseek_client.qobuz + qobuz = download_orchestrator.client("qobuz") result = qobuz.login_with_token(token) if result['status'] == 'success': + _sync_qobuz_credentials_to_worker() return jsonify({"success": True, **result}) else: return jsonify({"success": False, "error": result.get('message', 'Token login failed')}), 400 @@ -20178,7 +20351,7 @@ def qobuz_auth_token(): def qobuz_auth_status(): """Check if Qobuz client is authenticated.""" try: - qobuz = soulseek_client.qobuz + qobuz = download_orchestrator.client("qobuz") authenticated = qobuz.is_authenticated() user_info = {} if authenticated and qobuz.user_info: @@ -20195,7 +20368,8 @@ def qobuz_auth_status(): def qobuz_auth_logout(): """Logout from Qobuz.""" try: - soulseek_client.qobuz.logout() + download_orchestrator.client("qobuz").logout() + _sync_qobuz_credentials_to_worker() return jsonify({"success": True}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @@ -20675,8 +20849,8 @@ def _pause_enrichment_workers(label='discovery'): worker.pause() was_running[name] = True logger.warning(f"Paused {name} enrichment worker during {label}") - except Exception: - pass + except Exception as e: + logger.debug("enrichment worker pause failed: %s", e) return was_running @@ -20693,8 +20867,8 @@ def _resume_enrichment_workers(was_running, label='discovery'): if was_running.get(name) and worker: worker.resume() logger.info(f"Resumed {name} enrichment worker after {label}") - except Exception: - pass + except Exception as e: + logger.debug("enrichment worker resume failed: %s", e) def _sync_discovery_results_to_mirrored(source_type, source_playlist_id, discovery_results, discovery_source, profile_id=1): @@ -21136,9 +21310,7 @@ def _get_metadata_fallback_client(): def get_deezer_arl_status(): """Check if Deezer ARL is configured and authenticated.""" try: - deezer_dl = None - if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl: - deezer_dl = soulseek_client.deezer_dl + deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None if deezer_dl and deezer_dl.is_authenticated(): user_data = deezer_dl._user_data or {} return jsonify({ @@ -21155,9 +21327,7 @@ def get_deezer_arl_status(): def get_deezer_arl_playlists(): """Fetch user playlists via Deezer ARL authentication (like /api/spotify/playlists).""" try: - deezer_dl = None - if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl: - deezer_dl = soulseek_client.deezer_dl + deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None if not deezer_dl or not deezer_dl.is_authenticated(): return jsonify({'error': 'Deezer ARL not authenticated. Configure your ARL token in Settings > Downloads.'}), 401 @@ -21185,9 +21355,7 @@ def get_deezer_arl_playlists(): def get_deezer_arl_playlist_tracks(playlist_id): """Fetch full playlist with tracks via ARL (like /api/spotify/playlist/).""" try: - deezer_dl = None - if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl: - deezer_dl = soulseek_client.deezer_dl + deezer_dl = download_orchestrator.client("deezer_dl") if download_orchestrator and hasattr(download_orchestrator, 'client') else None if not deezer_dl or not deezer_dl.is_authenticated(): return jsonify({'error': 'Deezer ARL not authenticated.'}), 401 @@ -23276,8 +23444,7 @@ def _build_sync_deps(): return _discovery_sync.SyncDeps( config_manager=config_manager, sync_service=sync_service, - plex_client=plex_client, - jellyfin_client=jellyfin_client, + media_server_engine=media_server_engine, automation_engine=automation_engine, run_async=run_async, record_sync_history_start=_record_sync_history_start, @@ -23409,12 +23576,12 @@ def test_database_access(): # Test media clients logger.info(" Media clients status:") - logger.info(f" plex_client: {plex_client is not None}") - if plex_client: - logger.info(f" plex_client.is_connected(): {plex_client.is_connected()}") - logger.info(f" jellyfin_client: {jellyfin_client is not None}") - if jellyfin_client: - logger.info(f" jellyfin_client.is_connected(): {jellyfin_client.is_connected()}") + logger.info(f" media_server_engine.client('plex'): {media_server_engine.client('plex') is not None}") + if media_server_engine.client('plex'): + logger.info(f" media_server_engine.client('plex').is_connected(): {media_server_engine.client('plex').is_connected()}") + logger.info(f" media_server_engine.client('jellyfin'): {media_server_engine.client('jellyfin') is not None}") + if media_server_engine.client('jellyfin'): + logger.info(f" media_server_engine.client('jellyfin').is_connected(): {media_server_engine.client('jellyfin').is_connected()}") return jsonify({ "success": True, @@ -23423,8 +23590,8 @@ def test_database_access(): "database_initialized": db is not None, "database_stats": stats, "active_server": active_server, - "plex_connected": plex_client.is_connected() if plex_client else False, - "jellyfin_connected": jellyfin_client.is_connected() if jellyfin_client else False, + "plex_connected": media_server_engine.client('plex').is_connected() if media_server_engine.client('plex') else False, + "jellyfin_connected": media_server_engine.client('jellyfin').is_connected() if media_server_engine.client('jellyfin') else False, } }) @@ -24699,8 +24866,8 @@ def add_to_watchlist(): elif row['discogs_id']: artist_id = row['discogs_id'] source = 'discogs' - except Exception: - pass + except Exception as e: + logger.debug("watchlist artist source lookup failed: %s", e) if not source: fallback_source = _get_metadata_fallback_source() source = fallback_source if is_numeric_id else 'spotify' @@ -24780,16 +24947,16 @@ def add_to_watchlist(): try: pid = get_current_profile_id() socketio.emit('watchlist:count', _build_watchlist_count_payload(profile_id=pid), room=f'profile:{pid}') - except Exception: - pass + except Exception as e: + logger.debug("watchlist count emit failed: %s", e) try: if automation_engine: automation_engine.emit('watchlist_artist_added', { 'artist': artist_name, 'artist_id': str(artist_id), }) - except Exception: - pass + except Exception as e: + logger.debug("watchlist_artist_added emit failed: %s", e) _artmap_cache_invalidate(get_current_profile_id()) return jsonify({"success": True, "message": f"Added {artist_name} to watchlist"}) else: @@ -24817,16 +24984,16 @@ def remove_from_watchlist(): try: pid = get_current_profile_id() socketio.emit('watchlist:count', _build_watchlist_count_payload(profile_id=pid), room=f'profile:{pid}') - except Exception: - pass + except Exception as e: + logger.debug("watchlist count emit failed: %s", e) try: if automation_engine: automation_engine.emit('watchlist_artist_removed', { 'artist': data.get('artist_name', str(artist_id)), 'artist_id': str(artist_id), }) - except Exception: - pass + except Exception as e: + logger.debug("watchlist_artist_removed emit failed: %s", e) _artmap_cache_invalidate(get_current_profile_id()) return jsonify({"success": True, "message": "Removed artist from watchlist"}) else: @@ -24968,8 +25135,8 @@ def watchlist_all_unwatched_library_artists(): if artist.get('image_url'): try: database.update_watchlist_artist_image(artist_id, artist['image_url']) - except Exception: - pass + except Exception as e: + logger.debug("watchlist artist image update failed: %s", e) total_unwatched = len(unwatched_artists) message_parts = [f"Added {added} artist{'s' if added != 1 else ''} to watchlist"] @@ -25128,8 +25295,8 @@ def start_watchlist_scan(): try: if config_manager.get('discogs.token', ''): providers_to_backfill.append('discogs') - except Exception: - pass + except Exception as e: + logger.debug("discogs token backfill check failed: %s", e) for _bf_provider in providers_to_backfill: try: logger.debug(f"Checking for missing {_bf_provider} IDs in watchlist...") @@ -25292,8 +25459,8 @@ def start_watchlist_scan(): # Clear one-time rescan cutoff after full scan cycle try: scanner._clear_rescan_cutoff() - except Exception: - pass + except Exception as e: + logger.debug("scanner rescan cutoff clear failed: %s", e) # Always reset flag when scan completes (success or error) with watchlist_timer_lock: @@ -26130,8 +26297,8 @@ def enrich_similar_artists(): aid, fallback_source, image_url=img_url, genres=genres, popularity=0 ) - except Exception: - pass + except Exception as e: + logger.debug("similar artist enrichment failed: %s", e) cached_count = len(enriched) - len([aid for aid in uncached_ids if aid in enriched]) api_count = len([aid for aid in uncached_ids if aid in enriched]) @@ -26298,10 +26465,10 @@ def get_discover_recent_releases(): if album_id: try: database.update_discovery_recent_album_cover(album_id, cover) - except Exception: - pass - except Exception: - pass + except Exception as e: + logger.debug("recent album cover update failed: %s", e) + except Exception as e: + logger.debug("recent album cover fetch failed: %s", e) # Filter out blacklisted artists blacklisted = database.get_discovery_blacklist_names() @@ -26435,8 +26602,8 @@ def get_discover_because_you_listen_to(): if row and row[0]: artist_image = fix_artist_image_url(row[0]) conn.close() - except Exception: - pass + except Exception as e: + logger.debug("artist image lookup failed: %s", e) sections.append({ 'artist_name': artist_name, @@ -26621,8 +26788,8 @@ def resolve_cache_album(): if results: r = results[0] return jsonify({'success': True, 'entity_id': str(r.id), 'source': _get_metadata_fallback_source()}) - except Exception: - pass + except Exception as e: + logger.debug("fallback album search failed: %s", e) return jsonify({'success': False, 'error': 'Album not found in cache'}) except Exception as e: @@ -26937,8 +27104,8 @@ def get_seasonal_playlist(season_key): try: import json track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) - except: - pass + except Exception as e: + logger.debug("track_data_json parse: %s", e) tracks.append(track_dict) else: # Try discovery_pool as fallback (filtered by source) @@ -26964,8 +27131,8 @@ def get_seasonal_playlist(season_key): try: import json track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) - except: - pass + except Exception as e: + logger.debug("discovery track_data_json parse: %s", e) tracks.append(track_dict) config = SEASONAL_CONFIG[season_key] @@ -27370,8 +27537,8 @@ def get_your_artists_sources(): try: if tidal_client and hasattr(tidal_client, '_ensure_valid_token') and tidal_client._ensure_valid_token(): connected.append('tidal') - except Exception: - pass + except Exception as e: + logger.debug("tidal auth check failed: %s", e) # Last.fm if config_manager.get('lastfm.api_key', '') and config_manager.get('lastfm.session_key', ''): connected.append('lastfm') @@ -27379,12 +27546,12 @@ def get_your_artists_sources(): try: deezer_cl = _get_deezer_client() deezer_oauth = deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated() - deezer_arl = (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl - and soulseek_client.deezer_dl.is_authenticated()) + deezer_arl = (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl") + and download_orchestrator.client("deezer_dl").is_authenticated()) if deezer_oauth or deezer_arl: connected.append('deezer') - except Exception: - pass + except Exception as e: + logger.debug("deezer auth check failed: %s", e) return jsonify({"success": True, "enabled": enabled, "connected": connected}) except Exception as e: @@ -27501,10 +27668,10 @@ def _fetch_and_match_liked_artists(profile_id: int): if deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated(): logger.info("[Your Artists] Fetching favorite artists from Deezer (OAuth)...") artists = deezer_cl.get_user_favorite_artists(limit=200) - elif (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl - and soulseek_client.deezer_dl.is_authenticated()): + elif (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl") + and download_orchestrator.client("deezer_dl").is_authenticated()): logger.info("[Your Artists] Fetching favorite artists from Deezer (ARL)...") - artists = soulseek_client.deezer_dl.get_user_favorite_artists(limit=200) + artists = download_orchestrator.client("deezer_dl").get_user_favorite_artists(limit=200) for a in artists: database.upsert_liked_artist( artist_name=a['name'], source_service='deezer', @@ -27646,17 +27813,26 @@ def get_your_albums_sources(): try: if tidal_client and hasattr(tidal_client, '_ensure_valid_token') and tidal_client._ensure_valid_token(): connected.append('tidal') - except Exception: - pass + except Exception as e: + logger.debug("tidal auth check failed: %s", e) try: deezer_cl = _get_deezer_client() deezer_oauth = deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated() - deezer_arl = (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl - and soulseek_client.deezer_dl.is_authenticated()) + deezer_arl = (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl") + and download_orchestrator.client("deezer_dl").is_authenticated()) if deezer_oauth or deezer_arl: connected.append('deezer') - except Exception: - pass + except Exception as e: + logger.debug("deezer auth check failed: %s", e) + + # Discogs: counts as "connected" when a personal access token is + # configured. Username comes from /oauth/identity at fetch time; + # not required up front. + try: + if config_manager.get('discogs.token', ''): + connected.append('discogs') + except Exception as e: + logger.debug("discogs token check failed: %s", e) return jsonify({"success": True, "enabled": enabled, "connected": connected}) except Exception as e: @@ -27750,10 +27926,10 @@ def _fetch_liked_albums(profile_id: int): if deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated(): logger.info("[Your Albums] Fetching favorite albums from Deezer (OAuth)...") albums = deezer_cl.get_user_favorite_albums(limit=500) - elif (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl - and soulseek_client.deezer_dl.is_authenticated()): + elif (hasattr(download_orchestrator, 'client') and download_orchestrator.client("deezer_dl") + and download_orchestrator.client("deezer_dl").is_authenticated()): logger.info("[Your Albums] Fetching favorite albums from Deezer (ARL)...") - albums = soulseek_client.deezer_dl.get_user_favorite_albums(limit=500) + albums = download_orchestrator.client("deezer_dl").get_user_favorite_albums(limit=500) for a in albums: database.upsert_liked_album( album_name=a['album_name'], artist_name=a['artist_name'], @@ -27768,6 +27944,38 @@ def _fetch_liked_albums(profile_id: int): except Exception as e: logger.error(f"[Your Albums] Deezer fetch error: {e}") + # 4. Fetch from Discogs (user's collection) — uses personal access + # token from `discogs.token` config. Username resolved via the + # `/oauth/identity` endpoint at fetch time. Discogs is physical- + # media-first so many releases won't have streaming equivalents, + # but the click-context dispatch in the frontend opens the Discogs + # release detail and the user can manually trigger a download + # search if a digital match exists. + try: + if 'discogs' not in enabled_sources: + logger.warning("[Your Albums] Discogs skipped (disabled in sources config)") + elif not config_manager.get('discogs.token', ''): + logger.info("[Your Albums] Discogs skipped (no token configured)") + else: + from core.discogs_client import DiscogsClient + discogs_cl = DiscogsClient() + if discogs_cl.is_authenticated(): + logger.info("[Your Albums] Fetching collection from Discogs...") + releases = discogs_cl.get_user_collection() + for r in releases: + database.upsert_liked_album( + album_name=r['album_name'], artist_name=r['artist_name'], + source_service='discogs', + source_id=str(r['release_id']), source_id_type='discogs', + image_url=r.get('image_url'), release_date=r.get('release_date', ''), + total_tracks=r.get('total_tracks', 0), profile_id=profile_id + ) + fetched += len(releases) + if releases: + logger.info(f"[Your Albums] Fetched {len(releases)} from Discogs") + except Exception as e: + logger.error(f"[Your Albums] Discogs fetch error: {e}") + logger.info(f"[Your Albums] Total fetched: {fetched}") @@ -27807,8 +28015,8 @@ def get_your_artist_info(artist_id): 'lastfm_playcount': r.get('lastfm_playcount', 0), }) return jsonify(result) - except Exception: - pass + except Exception as e: + logger.debug("library artist lookup failed: %s", e) # 2. Try metadata cache try: @@ -27829,8 +28037,8 @@ def get_your_artist_info(artist_id): 'followers': cached.get('followers', {}).get('total', 0) if isinstance(cached.get('followers'), dict) else cached.get('followers', 0), }) return jsonify(result) - except Exception: - pass + except Exception as e: + logger.debug("metadata cache lookup failed: %s", e) # 3. Try Spotify API directly (genres, image, followers) try: @@ -27900,7 +28108,7 @@ def image_proxy(): 'img.discogs.com', 'i.discogs.com', # Discogs 'localhost', '127.0.0.1', 'host.docker.internal', # Local/Docker media servers ] - if not any(host == h or host.endswith('.' + h) for h in allowed_hosts) and not _is_internal_image_host(url): + if not any(host == h or host.endswith('.' + h) for h in allowed_hosts) and not is_internal_image_host(url): return '', 403 try: resp = requests.get(url, timeout=10, stream=True, headers={ @@ -29102,17 +29310,17 @@ def start_metadata_update(): # Get appropriate media client - Support all three servers if active_server == "jellyfin": - media_client = jellyfin_client + media_client = media_server_engine.client('jellyfin') if not media_client: add_activity_item("", "Metadata Update", "Jellyfin client not available", "Now") return jsonify({"success": False, "error": "Jellyfin client not available"}), 400 elif active_server == "navidrome": - media_client = navidrome_client + media_client = media_server_engine.client('navidrome') if not media_client: add_activity_item("", "Metadata Update", "Navidrome client not available", "Now") return jsonify({"success": False, "error": "Navidrome client not available"}), 400 else: # plex - media_client = plex_client + media_client = media_server_engine.client('plex') if not media_client: add_activity_item("", "Metadata Update", "Plex client not available", "Now") return jsonify({"success": False, "error": "Plex client not available"}), 400 @@ -31296,8 +31504,8 @@ def mirror_playlist_endpoint(): 'source': source, 'track_count': str(len(tracks)), }) - except Exception: - pass + except Exception as e: + logger.debug("mirrored_playlist_created emit failed: %s", e) return jsonify({"success": True, "playlist_id": playlist_id}) except Exception as e: @@ -31471,8 +31679,8 @@ def fix_discovery_pool_track(): cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, row['track_name'], row['artist_name'] ) - except Exception: - pass + except Exception as e: + logger.debug("discovery cache match save failed: %s", e) return jsonify({"success": True}) except Exception as e: @@ -31896,8 +32104,8 @@ def playlist_explorer_album_tracks(album_id): if cache and tracks: try: cache.store_entity(source_name, 'album_tracks', cache_key, result) - except Exception: - pass + except Exception as e: + logger.debug("album_tracks cache store failed: %s", e) return jsonify(result) except Exception as e: @@ -32040,7 +32248,7 @@ def start_oauth_callback_servers(): _clear_rate_limit() spotify_client._invalidate_auth_cache() # Invalidate status cache so next poll picks up the new connection - _status_cache_timestamps['spotify'] = 0 + invalidate_metadata_status_caches() # Refresh enrichment worker's client so it picks up new auth if spotify_enrichment_worker and hasattr(spotify_enrichment_worker, 'client'): spotify_enrichment_worker.client.reload_config() @@ -32053,7 +32261,7 @@ def start_oauth_callback_servers(): else: _oauth_logger.warning("Spotify token exchange succeeded but authentication validation failed") spotify_client._invalidate_auth_cache() - _status_cache_timestamps['spotify'] = 0 + invalidate_metadata_status_caches() add_activity_item("", "Spotify Auth Warning", "OAuth completed, but Spotify did not confirm an authenticated session", "Now") self.send_response(200) self.send_header('Content-type', 'text/html') @@ -32106,8 +32314,8 @@ def start_oauth_callback_servers(): self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(f'

Internal Server Error

{str(e)}

'.encode()) - except Exception: - pass # Connection already broken, nothing more we can do + except Exception as _e: + _oauth_logger.debug("oauth response write: %s", _e) def log_message(self, format, *args): pass # Suppress BaseHTTPRequestHandler access logs (we use our own logger) @@ -32248,57 +32456,9 @@ except Exception as e: logger.error(f"MusicBrainz worker initialization failed: {e}") mb_worker = None -# --- MusicBrainz API Endpoints --- - -@app.route('/api/musicbrainz/status', methods=['GET']) -def musicbrainz_status(): - """Get MusicBrainz enrichment status for UI polling""" - try: - if mb_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = mb_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting MusicBrainz status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/musicbrainz/pause', methods=['POST']) -def musicbrainz_pause(): - """Pause MusicBrainz enrichment worker (finishes current match first)""" - try: - if mb_worker is None: - return jsonify({'error': 'MusicBrainz worker not initialized'}), 400 - - mb_worker.pause() - config_manager.set('musicbrainz_enrichment_paused', True) - logger.info("MusicBrainz worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing MusicBrainz worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/musicbrainz/resume', methods=['POST']) -def musicbrainz_resume(): - """Resume MusicBrainz enrichment worker""" - try: - if mb_worker is None: - return jsonify({'error': 'MusicBrainz worker not initialized'}), 400 - - mb_worker.resume() - config_manager.set('musicbrainz_enrichment_paused', False) - logger.info("MusicBrainz worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming MusicBrainz worker: {e}") - return jsonify({'error': str(e)}), 500 +# MusicBrainz status / pause / resume routes are now served by the +# generic enrichment blueprint registered in core/enrichment/api.py +# under /api/enrichment/musicbrainz/{status,pause,resume}. # ================================================================================================ # END MUSICBRAINZ INTEGRATION @@ -32325,57 +32485,8 @@ except Exception as e: logger.error(f"AudioDB worker initialization failed: {e}") audiodb_worker = None -# --- AudioDB API Endpoints --- - -@app.route('/api/audiodb/status', methods=['GET']) -def audiodb_status(): - """Get AudioDB enrichment status for UI polling""" - try: - if audiodb_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = audiodb_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting AudioDB status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/audiodb/pause', methods=['POST']) -def audiodb_pause(): - """Pause AudioDB enrichment worker""" - try: - if audiodb_worker is None: - return jsonify({'error': 'AudioDB worker not initialized'}), 400 - - audiodb_worker.pause() - config_manager.set('audiodb_enrichment_paused', True) - logger.info("AudioDB worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing AudioDB worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/audiodb/resume', methods=['POST']) -def audiodb_resume(): - """Resume AudioDB enrichment worker""" - try: - if audiodb_worker is None: - return jsonify({'error': 'AudioDB worker not initialized'}), 400 - - audiodb_worker.resume() - config_manager.set('audiodb_enrichment_paused', False) - logger.info("AudioDB worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming AudioDB worker: {e}") - return jsonify({'error': str(e)}), 500 +# AudioDB status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/audiodb/{status,pause,resume}. # ================================================================================================ # END AUDIODB INTEGRATION @@ -32398,47 +32509,8 @@ except Exception as e: logger.error(f"Discogs worker initialization failed: {e}") discogs_worker = None -# --- Discogs API Endpoints --- - -@app.route('/api/discogs/status', methods=['GET']) -def discogs_status(): - """Get Discogs enrichment status for UI polling""" - try: - if discogs_worker is None: - return jsonify({ - 'enabled': False, 'running': False, 'paused': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - }), 200 - return jsonify(discogs_worker.get_stats()), 200 - except Exception as e: - return jsonify({'error': str(e)}), 500 - -@app.route('/api/discogs/pause', methods=['POST']) -def discogs_pause(): - """Pause Discogs enrichment worker""" - try: - if discogs_worker is None: - return jsonify({'error': 'Discogs worker not initialized'}), 400 - discogs_worker.pause() - config_manager.set('discogs_enrichment_paused', True) - logger.info("Discogs worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - return jsonify({'error': str(e)}), 500 - -@app.route('/api/discogs/resume', methods=['POST']) -def discogs_resume(): - """Resume Discogs enrichment worker""" - try: - if discogs_worker is None: - return jsonify({'error': 'Discogs worker not initialized'}), 400 - discogs_worker.resume() - config_manager.set('discogs_enrichment_paused', False) - logger.info("Discogs worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - return jsonify({'error': str(e)}), 500 +# Discogs status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/discogs/{status,pause,resume}. # ================================================================================================ # DEEZER ENRICHMENT INTEGRATION @@ -32460,57 +32532,8 @@ except Exception as e: logger.error(f"Deezer worker initialization failed: {e}") deezer_worker = None -# --- Deezer API Endpoints --- - -@app.route('/api/deezer/status', methods=['GET']) -def deezer_status(): - """Get Deezer enrichment status for UI polling""" - try: - if deezer_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = deezer_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting Deezer status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/deezer/pause', methods=['POST']) -def deezer_pause(): - """Pause Deezer enrichment worker""" - try: - if deezer_worker is None: - return jsonify({'error': 'Deezer worker not initialized'}), 400 - - deezer_worker.pause() - config_manager.set('deezer_enrichment_paused', True) - logger.info("Deezer worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing Deezer worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/deezer/resume', methods=['POST']) -def deezer_resume(): - """Resume Deezer enrichment worker""" - try: - if deezer_worker is None: - return jsonify({'error': 'Deezer worker not initialized'}), 400 - - deezer_worker.resume() - config_manager.set('deezer_enrichment_paused', False) - logger.info("Deezer worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming Deezer worker: {e}") - return jsonify({'error': str(e)}), 500 +# Deezer status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/deezer/{status,pause,resume}. # ================================================================================================ # END DEEZER INTEGRATION @@ -32569,66 +32592,10 @@ def get_rate_monitor_history(service_key): except Exception as e: return jsonify({'error': str(e)}), 500 -# --- Spotify API Endpoints --- - -@app.route('/api/spotify-enrichment/status', methods=['GET']) -def spotify_enrichment_status(): - """Get Spotify enrichment status for UI polling""" - try: - if spotify_enrichment_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = spotify_enrichment_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting Spotify enrichment status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/spotify-enrichment/pause', methods=['POST']) -def spotify_enrichment_pause(): - """Pause Spotify enrichment worker""" - try: - if spotify_enrichment_worker is None: - return jsonify({'error': 'Spotify enrichment worker not initialized'}), 400 - - spotify_enrichment_worker.pause() - config_manager.set('spotify_enrichment_paused', True) - # Drop any auto-pause marker so the post-download resume loop won't - # override this explicit user pause. - _download_auto_paused.discard('spotify-enrichment') - logger.info("Spotify enrichment worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing Spotify enrichment worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/spotify-enrichment/resume', methods=['POST']) -def spotify_enrichment_resume(): - """Resume Spotify enrichment worker""" - try: - if spotify_enrichment_worker is None: - return jsonify({'error': 'Spotify enrichment worker not initialized'}), 400 - - # Block resume while Spotify is rate limited - if _spotify_rate_limited(): - return jsonify({'error': 'Cannot resume while Spotify is rate limited', 'rate_limited': True}), 429 - - spotify_enrichment_worker.resume() - config_manager.set('spotify_enrichment_paused', False) - _download_auto_paused.discard('spotify-enrichment') - _download_yield_override.add('spotify-enrichment') # User override — don't re-pause during this download session - logger.info("Spotify enrichment worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming Spotify enrichment worker: {e}") - return jsonify({'error': str(e)}), 500 +# Spotify status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/spotify/{status,pause,resume}. +# The rate-limit guard, auto-pause token cleanup, and yield-override behavior +# are encoded on the EnrichmentService descriptor (see core/enrichment/services.py). # ================================================================================================ # END SPOTIFY ENRICHMENT INTEGRATION @@ -32655,57 +32622,8 @@ except Exception as e: logger.error(f"iTunes enrichment worker initialization failed: {e}") itunes_enrichment_worker = None -# --- iTunes API Endpoints --- - -@app.route('/api/itunes-enrichment/status', methods=['GET']) -def itunes_enrichment_status(): - """Get iTunes enrichment status for UI polling""" - try: - if itunes_enrichment_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = itunes_enrichment_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting iTunes enrichment status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/itunes-enrichment/pause', methods=['POST']) -def itunes_enrichment_pause(): - """Pause iTunes enrichment worker""" - try: - if itunes_enrichment_worker is None: - return jsonify({'error': 'iTunes enrichment worker not initialized'}), 400 - - itunes_enrichment_worker.pause() - config_manager.set('itunes_enrichment_paused', True) - logger.info("iTunes enrichment worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing iTunes enrichment worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/itunes-enrichment/resume', methods=['POST']) -def itunes_enrichment_resume(): - """Resume iTunes enrichment worker""" - try: - if itunes_enrichment_worker is None: - return jsonify({'error': 'iTunes enrichment worker not initialized'}), 400 - - itunes_enrichment_worker.resume() - config_manager.set('itunes_enrichment_paused', False) - logger.info("iTunes enrichment worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming iTunes enrichment worker: {e}") - return jsonify({'error': str(e)}), 500 +# iTunes status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/itunes/{status,pause,resume}. # ================================================================================================ # END ITUNES ENRICHMENT INTEGRATION @@ -32731,62 +32649,10 @@ except Exception as e: logger.error(f"Last.fm worker initialization failed: {e}") lastfm_worker = None -# --- Last.fm API Endpoints --- - -@app.route('/api/lastfm-enrichment/status', methods=['GET']) -def lastfm_enrichment_status(): - """Get Last.fm enrichment status for UI polling""" - try: - if lastfm_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = lastfm_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting Last.fm enrichment status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/lastfm-enrichment/pause', methods=['POST']) -def lastfm_enrichment_pause(): - """Pause Last.fm enrichment worker""" - try: - if lastfm_worker is None: - return jsonify({'error': 'Last.fm worker not initialized'}), 400 - - lastfm_worker.pause() - config_manager.set('lastfm_enrichment_paused', True) - # Drop any auto-pause marker so the post-download resume loop won't - # override this explicit user pause. - _download_auto_paused.discard('lastfm-enrichment') - logger.info("Last.fm worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing Last.fm worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/lastfm-enrichment/resume', methods=['POST']) -def lastfm_enrichment_resume(): - """Resume Last.fm enrichment worker""" - try: - if lastfm_worker is None: - return jsonify({'error': 'Last.fm worker not initialized'}), 400 - - lastfm_worker.resume() - config_manager.set('lastfm_enrichment_paused', False) - _download_auto_paused.discard('lastfm-enrichment') - _download_yield_override.add('lastfm-enrichment') # User override — don't re-pause during this download session - logger.info("Last.fm worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming Last.fm worker: {e}") - return jsonify({'error': str(e)}), 500 +# Last.fm status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/lastfm/{status,pause,resume}. +# The auto-pause token cleanup + yield-override behavior is encoded on the +# EnrichmentService descriptor (see core/enrichment/services.py). @app.route('/api/artist//lastfm-top-tracks', methods=['GET']) def get_artist_lastfm_top_tracks(artist_id): @@ -32879,60 +32745,10 @@ except Exception as e: # --- Genius API Endpoints --- -@app.route('/api/genius-enrichment/status', methods=['GET']) -def genius_enrichment_status(): - """Get Genius enrichment status for UI polling""" - try: - if genius_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = genius_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting Genius enrichment status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/genius-enrichment/pause', methods=['POST']) -def genius_enrichment_pause(): - """Pause Genius enrichment worker""" - try: - if genius_worker is None: - return jsonify({'error': 'Genius worker not initialized'}), 400 - - genius_worker.pause() - config_manager.set('genius_enrichment_paused', True) - # Drop any auto-pause marker so the post-download resume loop won't - # override this explicit user pause. - _download_auto_paused.discard('genius-enrichment') - logger.info("Genius worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing Genius worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/genius-enrichment/resume', methods=['POST']) -def genius_enrichment_resume(): - """Resume Genius enrichment worker""" - try: - if genius_worker is None: - return jsonify({'error': 'Genius worker not initialized'}), 400 - - genius_worker.resume() - config_manager.set('genius_enrichment_paused', False) - _download_auto_paused.discard('genius-enrichment') - _download_yield_override.add('genius-enrichment') # User override — don't re-pause during this download session - logger.info("Genius worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming Genius worker: {e}") - return jsonify({'error': str(e)}), 500 +# Genius status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/genius/{status,pause,resume}. +# The auto-pause token cleanup + yield-override behavior is encoded on the +# EnrichmentService descriptor (see core/enrichment/services.py). # ================================================================================================ # END GENIUS ENRICHMENT INTEGRATION @@ -32957,58 +32773,10 @@ except Exception as e: logger.error(f"Tidal worker initialization failed: {e}") tidal_enrichment_worker = None -# --- Tidal Enrichment API Endpoints --- - -@app.route('/api/tidal-enrichment/status', methods=['GET']) -def tidal_enrichment_status(): - """Get Tidal enrichment status for UI polling""" - try: - if tidal_enrichment_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'authenticated': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = tidal_enrichment_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting Tidal enrichment status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/tidal-enrichment/pause', methods=['POST']) -def tidal_enrichment_pause(): - """Pause Tidal enrichment worker""" - try: - if tidal_enrichment_worker is None: - return jsonify({'error': 'Tidal worker not initialized'}), 400 - - tidal_enrichment_worker.pause() - config_manager.set('tidal_enrichment_paused', True) - logger.info("Tidal worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing Tidal worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/tidal-enrichment/resume', methods=['POST']) -def tidal_enrichment_resume(): - """Resume Tidal enrichment worker""" - try: - if tidal_enrichment_worker is None: - return jsonify({'error': 'Tidal worker not initialized'}), 400 - - tidal_enrichment_worker.resume() - config_manager.set('tidal_enrichment_paused', False) - logger.info("Tidal worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming Tidal worker: {e}") - return jsonify({'error': str(e)}), 500 +# Tidal status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/tidal/{status,pause,resume}. +# The 'authenticated': False fallback field is encoded on the +# EnrichmentService descriptor (see core/enrichment/services.py). # ================================================================================================ # QOBUZ ENRICHMENT WORKER @@ -33043,58 +32811,10 @@ _init_service_search( audiodb_worker_obj=audiodb_worker, ) -# --- Qobuz Enrichment API Endpoints --- - -@app.route('/api/qobuz-enrichment/status', methods=['GET']) -def qobuz_enrichment_status(): - """Get Qobuz enrichment status for UI polling""" - try: - if qobuz_enrichment_worker is None: - return jsonify({ - 'enabled': False, - 'running': False, - 'paused': False, - 'authenticated': False, - 'current_item': None, - 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, - 'progress': {} - }), 200 - - status = qobuz_enrichment_worker.get_stats() - return jsonify(status), 200 - except Exception as e: - logger.error(f"Error getting Qobuz enrichment status: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/qobuz-enrichment/pause', methods=['POST']) -def qobuz_enrichment_pause(): - """Pause Qobuz enrichment worker""" - try: - if qobuz_enrichment_worker is None: - return jsonify({'error': 'Qobuz worker not initialized'}), 400 - - qobuz_enrichment_worker.pause() - config_manager.set('qobuz_enrichment_paused', True) - logger.info("Qobuz worker paused via UI") - return jsonify({'status': 'paused'}), 200 - except Exception as e: - logger.error(f"Error pausing Qobuz worker: {e}") - return jsonify({'error': str(e)}), 500 - -@app.route('/api/qobuz-enrichment/resume', methods=['POST']) -def qobuz_enrichment_resume(): - """Resume Qobuz enrichment worker""" - try: - if qobuz_enrichment_worker is None: - return jsonify({'error': 'Qobuz worker not initialized'}), 400 - - qobuz_enrichment_worker.resume() - config_manager.set('qobuz_enrichment_paused', False) - logger.info("Qobuz worker resumed via UI") - return jsonify({'status': 'running'}), 200 - except Exception as e: - logger.error(f"Error resuming Qobuz worker: {e}") - return jsonify({'error': str(e)}), 500 +# Qobuz status / pause / resume routes are now served by the +# generic enrichment blueprint at /api/enrichment/qobuz/{status,pause,resume}. +# The 'authenticated': False fallback field is encoded on the +# EnrichmentService descriptor (see core/enrichment/services.py). # ================================================================================================ # END TIDAL/QOBUZ ENRICHMENT INTEGRATION @@ -33130,7 +32850,7 @@ register_runtime_clients( ) _init_connection_test( - soulseek_client_obj=soulseek_client, + download_orchestrator_obj=download_orchestrator, qobuz_worker=qobuz_enrichment_worker, hydrabase_client_obj=hydrabase_client, docker_resolve_url_fn=docker_resolve_url, @@ -33143,12 +32863,12 @@ _init_discover_hero(get_metadata_fallback_client_fn=_get_metadata_fallback_clien _init_download_validation( matching_engine_obj=matching_engine, - soulseek_client_obj=soulseek_client, + download_orchestrator_obj=download_orchestrator, ) _init_wishlist_failed( engine=automation_engine, - soulseek_client_obj=soulseek_client, + download_orchestrator_obj=download_orchestrator, sweep_fn=_sweep_empty_download_directories, ) @@ -33167,7 +32887,7 @@ _init_debug_info( sync_states_dict=sync_states, youtube_playlist_states_dict=youtube_playlist_states, tidal_discovery_states_dict=tidal_discovery_states, - soulseek_client_obj=soulseek_client, + download_orchestrator_obj=download_orchestrator, log_path=_log_path, log_dir=_log_dir, flask_app=app, @@ -33186,7 +32906,7 @@ _init_download_monitor( start_next_batch_of_downloads=_start_next_batch_of_downloads, orphaned_download_keys=_orphaned_download_keys, missing_download_executor_obj=missing_download_executor, - soulseek_client_obj=soulseek_client, + download_orchestrator_obj=download_orchestrator, ) # --- Hydrabase Auto-Reconnect --- @@ -33305,9 +33025,7 @@ try: listening_stats_worker = ListeningStatsWorker( database=listening_stats_db, config_manager=config_manager, - plex_client=plex_client, - jellyfin_client=jellyfin_client, - navidrome_client=navidrome_client, + media_server_engine=media_server_engine, ) listening_stats_worker.start() logger.info("Listening stats worker initialized and started") @@ -33413,6 +33131,21 @@ def stats_db_storage(): except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 +@app.route('/api/stats/library-disk-usage', methods=['GET']) +def stats_library_disk_usage(): + """Library on-disk size + per-format breakdown. + + Reads `tracks.file_size` populated by the deep scan from data the + media server already returns. Returns ``has_data: false`` on fresh + installs that haven't run a deep scan since the migration — UI + shows "Run a Deep Scan to populate" in that case. + """ + try: + data = _stats_queries.get_library_disk_usage(get_database()) + return jsonify({'success': True, **data}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}), 500 + @app.route('/api/stats/recent', methods=['GET']) def stats_recent(): """Get recently played tracks.""" @@ -33516,12 +33249,14 @@ try: state['status'] = status state['progress'] = 100 state['finished_at'] = datetime.now(timezone.utc).isoformat() - summary = f'Done: {result.scanned} scanned, {result.auto_fixed} fixed, {result.findings_created} findings, {result.errors} errors' + skipped_dedup = getattr(result, 'findings_skipped_dedup', 0) or 0 + existing_part = f' ({skipped_dedup} already existed)' if skipped_dedup else '' + summary = f'Done: {result.scanned} scanned, {result.auto_fixed} fixed, {result.findings_created} findings{existing_part}, {result.errors} errors' state['log'].append({'type': 'success' if status == 'finished' else 'error', 'text': summary}) try: socketio.emit('repair:progress', {job_id: dict(state)}) - except Exception: - pass + except Exception as e: + logger.debug("repair progress emit failed: %s", e) repair_worker.register_progress_callbacks(_repair_job_start, _repair_job_progress, _repair_job_finish) # Store refs for WebSocket push loop @@ -34006,8 +33741,8 @@ def import_staging_hints(): if album: key = (album.strip(), (artist or '').strip()) tag_albums[key] = tag_albums.get(key, 0) + 1 - except Exception: - pass + except Exception as e: + logger.debug("tag read failed: %s", e) # Build search queries, prioritizing tag-based hints (more specific) queries = [] @@ -34163,8 +33898,8 @@ def import_album_process(): 'completed_tracks': str(processed), 'failed_tracks': str(len(errors)), }) - except Exception: - pass + except Exception as e: + logger.debug("album import automation emit failed: %s", e) # Rebuild suggestions cache since staging contents changed if processed > 0: @@ -34327,8 +34062,8 @@ def import_singles_process(): 'completed_tracks': str(processed), 'failed_tracks': str(len(errors)), }) - except Exception: - pass + except Exception as e: + logger.debug("singles import automation emit failed: %s", e) # Rebuild suggestions cache since staging contents changed if processed > 0: @@ -34505,17 +34240,7 @@ def _build_status_payload(): download_mode = config_manager.get('download_source.mode', 'hybrid') soulseek_data = dict(_status_cache.get('soulseek', {})) soulseek_data['source'] = download_mode - - # Always include fresh rate limit info (it changes over time as ban expires) - # Call is_rate_limited() first to ensure ban-end timestamp is recorded for cooldown - spotify_data = dict(_status_cache.get('spotify', {})) - is_rl = spotify_client.is_rate_limited() if spotify_client else False - rate_limit_info = spotify_client.get_rate_limit_info() if is_rl else None - cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0 - spotify_data['rate_limited'] = is_rl - spotify_data['rate_limit'] = rate_limit_info - if cooldown_remaining > 0: - spotify_data['post_ban_cooldown'] = cooldown_remaining + metadata_status = get_metadata_status_snapshot(spotify_client=spotify_client) # Count active downloads for nav badge active_dl_count = 0 @@ -34524,11 +34249,12 @@ def _build_status_payload(): for t in download_tasks.values(): if t.get('status') in ('downloading', 'searching', 'post_processing', 'queued', 'pending'): active_dl_count += 1 - except Exception: - pass + except Exception as e: + logger.debug("active download count failed: %s", e) return { - 'spotify': spotify_data, + 'metadata_source': metadata_status['metadata_source'], + 'spotify': metadata_status['spotify'], 'media_server': _status_cache.get('media_server', {}), 'soulseek': soulseek_data, 'active_media_server': config_manager.get_active_media_server(), @@ -34569,8 +34295,8 @@ def _hydrabase_reconnect_loop(): if _hydrabase_ws is not None and _hydrabase_ws.connected: _consecutive_failures = 0 continue - except Exception: - pass # Socket in bad state — treat as disconnected + except Exception as e: + logger.debug("hydrabase socket check: %s", e) # Disconnected with auto_connect enabled — try to reconnect # Back off: 30s, 60s, 120s, max 300s between attempts @@ -34584,8 +34310,8 @@ def _hydrabase_reconnect_loop(): if _hydrabase_ws: try: _hydrabase_ws.close() - except: - pass + except Exception as e: + logger.debug("hydrabase reconnect close: %s", e) ws = websocket.create_connection( hydra_cfg['url'], header={"x-api-key": hydra_cfg['api_key']}, @@ -34600,8 +34326,8 @@ def _hydrabase_reconnect_loop(): logger.error(f"[Hydrabase] Reconnect attempt failed ({_consecutive_failures}): {e}") elif _consecutive_failures == 4: logger.error("[Hydrabase] Reconnect failing repeatedly — suppressing further logs until success") - except Exception: - pass # Don't crash the monitor loop + except Exception as e: + logger.debug("hydrabase monitor loop: %s", e) def _emit_service_status_loop(): """Background thread that pushes service status every 5 seconds.""" @@ -34744,8 +34470,8 @@ def _has_active_downloads(): for batch_data in download_batches.values(): if batch_data.get('phase') not in ('complete', 'error', 'cancelled', None): return True - except Exception: - pass + except Exception as e: + logger.debug("active downloads check failed: %s", e) return False @@ -34754,6 +34480,104 @@ _download_auto_paused = set() _download_yield_override = set() # Workers the user explicitly resumed during downloads — don't re-pause +# --------------------------------------------------------------------------- +# Enrichment service registry +# --------------------------------------------------------------------------- +# Generic ``/api/enrichment//{status,pause,resume}`` routes that +# replace 30 near-identical per-service routes scattered through this file. +# The old per-service routes still exist below as a fallback during the +# soak period; PR-2 deletes them once the dashboard has cut over to the +# generic ones. See `core/enrichment/services.py` for the registry. +from core.enrichment.api import ( + configure as _configure_enrichment_api, + create_blueprint as _create_enrichment_blueprint, +) +from core.enrichment.services import ( + EnrichmentService as _EnrichmentService, + register_services as _register_enrichment_services, +) + + +def _spotify_resume_pre_check(): + """Mirror the inline Spotify rate-limit guard from the legacy + ``/api/spotify-enrichment/resume`` route. Returns + ``(429, message)`` to short-circuit when banned, ``None`` when ok.""" + try: + if _spotify_rate_limited(): + return (429, 'Cannot resume while Spotify is rate limited') + except Exception as e: + logger.debug("spotify rate-limit pre-check failed: %s", e) + return None + + +_register_enrichment_services([ + _EnrichmentService( + id='musicbrainz', display_name='MusicBrainz', + worker_getter=lambda: mb_worker, + config_paused_key='musicbrainz_enrichment_paused', + ), + _EnrichmentService( + id='audiodb', display_name='AudioDB', + worker_getter=lambda: audiodb_worker, + config_paused_key='audiodb_enrichment_paused', + ), + _EnrichmentService( + id='discogs', display_name='Discogs', + worker_getter=lambda: discogs_worker, + config_paused_key='discogs_enrichment_paused', + ), + _EnrichmentService( + id='deezer', display_name='Deezer', + worker_getter=lambda: deezer_worker, + config_paused_key='deezer_enrichment_paused', + ), + _EnrichmentService( + id='spotify', display_name='Spotify', + worker_getter=lambda: spotify_enrichment_worker, + config_paused_key='spotify_enrichment_paused', + pre_resume_check=_spotify_resume_pre_check, + auto_pause_token='spotify-enrichment', + ), + _EnrichmentService( + id='itunes', display_name='iTunes', + worker_getter=lambda: itunes_enrichment_worker, + config_paused_key='itunes_enrichment_paused', + ), + _EnrichmentService( + id='lastfm', display_name='Last.fm', + worker_getter=lambda: lastfm_worker, + config_paused_key='lastfm_enrichment_paused', + auto_pause_token='lastfm-enrichment', + ), + _EnrichmentService( + id='genius', display_name='Genius', + worker_getter=lambda: genius_worker, + config_paused_key='genius_enrichment_paused', + auto_pause_token='genius-enrichment', + ), + _EnrichmentService( + id='tidal', display_name='Tidal', + worker_getter=lambda: tidal_enrichment_worker, + config_paused_key='tidal_enrichment_paused', + extra_status_defaults={'authenticated': False}, + ), + _EnrichmentService( + id='qobuz', display_name='Qobuz', + worker_getter=lambda: qobuz_enrichment_worker, + config_paused_key='qobuz_enrichment_paused', + extra_status_defaults={'authenticated': False}, + ), +]) + +_configure_enrichment_api( + config_set=lambda key, value: config_manager.set(key, value), + auto_paused_discard=lambda token: _download_auto_paused.discard(token), + yield_override_add=lambda token: _download_yield_override.add(token), +) + +app.register_blueprint(_create_enrichment_blueprint()) + + def _emit_rate_monitor_loop(): """Background thread that pushes API call rate data every 1 second for speedometer gauges. Also includes enrichment worker status so the combined cards have everything.""" @@ -34789,19 +34613,19 @@ def _emit_rate_monitor_loop(): } if svc_key == 'spotify' and enr.get('daily_budget'): entry['worker']['daily_budget'] = enr['daily_budget'] - except Exception: - pass + except Exception as e: + logger.debug("enrichment worker status build failed: %s", e) # Add Spotify rate limit state try: - if spotify_client: - rl_info = spotify_client.get_rate_limit_info() - if rl_info: - payload['spotify']['rate_limited'] = True - payload['spotify']['rl_remaining'] = rl_info.get('remaining_seconds', 0) - payload['spotify']['rl_endpoint'] = rl_info.get('endpoint', '') - except Exception: - pass + spotify_status = get_spotify_status(spotify_client=spotify_client) + rl_info = spotify_status.get('rate_limit') + if spotify_status.get('rate_limited') and rl_info: + payload['spotify']['rate_limited'] = True + payload['spotify']['rl_remaining'] = rl_info.get('remaining_seconds', 0) + payload['spotify']['rl_endpoint'] = rl_info.get('endpoint', '') + except Exception as e: + logger.debug("spotify rate-limit status read failed: %s", e) socketio.emit('rate-monitor:update', payload) except Exception as e: @@ -35041,8 +34865,8 @@ def _emit_sync_progress_loop(): socketio.emit('sync:progress', { 'playlist_id': pid, **state }, room=f'sync:{pid}') - except Exception: - pass + except Exception as e: + logger.debug("sync progress emit failed: %s", e) except Exception as e: logger.debug(f"Error in sync progress loop: {e}") @@ -35079,8 +34903,8 @@ def _emit_discovery_progress_loop(): 'complete': state.get('phase') == 'discovered', } socketio.emit('discovery:progress', payload, room=f'discovery:{pid}') - except Exception: - pass + except Exception as e: + logger.debug("discovery progress emit failed: %s", e) except Exception as e: logger.debug(f"Error in {platform_name} discovery loop: {e}") @@ -35223,8 +35047,8 @@ def start_runtime_services(): logger.info("Automation engine started") try: automation_engine.emit('app_started', {}) - except Exception: - pass + except Exception as e: + logger.debug("app_started emit failed: %s", e) except AttributeError as e: logger.error(f"Automation engine failed to start: {e}") logger.info(" If using Docker, check that your volume mount is /app/data (not /app/database)") diff --git a/webui/index.html b/webui/index.html index 31f00e67..3de06f8e 100644 --- a/webui/index.html +++ b/webui/index.html @@ -279,17 +279,17 @@

Service Status

-
+
- Spotify + Metadata Source
-
+
- Plex + Media Server
-
+
- Soulseek + Download Source
@@ -616,22 +616,22 @@

Service Status

-
+
- Spotify + Metadata Source ● + id="metadata-source-status-indicator">●
-

Disconnected

-

Response: --

+

Disconnected

+

Response: --

+ onclick="testDashboardConnection(getActiveMetadataSource())">Test Connection
-
+
- Server + Media Server ●
@@ -642,9 +642,9 @@ Connection
-
+
- Soulseek + Download Source ●
@@ -2431,8 +2431,9 @@
@@ -2913,48 +2914,6 @@
- - -
@@ -4365,6 +4324,7 @@ +
@@ -4412,8 +4372,8 @@
- - + +
@@ -4675,6 +4635,28 @@
+ + +
@@ -5217,13 +5201,14 @@ - 4 tags + 5 tags
@@ -5275,6 +5260,24 @@ + +
+
+ ▶ + + 5 tags +
+ +
+
@@ -6011,6 +6014,18 @@
+ +
+
Library Disk Usage
+
+
+
—
+
Run a Deep Scan to populate
+
+
+
+
+
Database Storage
@@ -6342,7 +6357,7 @@ - diff --git a/webui/static/api-monitor.js b/webui/static/api-monitor.js index c900d824..01e27f0a 100644 --- a/webui/static/api-monitor.js +++ b/webui/static/api-monitor.js @@ -1837,6 +1837,19 @@ function startWatchlistCountdownTimer(initialSeconds) { } let remainingSeconds = initialSeconds; + const timerElement = document.getElementById('watchlist-next-auto-timer'); + const updateTimerText = (seconds) => { + if (!timerElement) return; + const countdownText = seconds > 0 ? formatCountdownTime(seconds) : ''; + timerElement.textContent = `Next Auto${countdownText ? ': ' + countdownText : ''}`; + }; + + // If there is no scheduled next run, don't keep polling the endpoint every second. + if (remainingSeconds <= 0) { + updateTimerText(0); + watchlistCountdownInterval = null; + return; + } watchlistCountdownInterval = setInterval(async () => { remainingSeconds--; @@ -1846,23 +1859,23 @@ function startWatchlistCountdownTimer(initialSeconds) { try { const response = await fetch('/api/watchlist/count'); const data = await response.json(); - remainingSeconds = data.next_run_in_seconds || 0; + const nextRunSeconds = data.next_run_in_seconds || 0; - const timerElement = document.getElementById('watchlist-next-auto-timer'); - if (timerElement) { - const countdownText = formatCountdownTime(remainingSeconds); - timerElement.textContent = `Next Auto${countdownText ? ': ' + countdownText : ''}`; + if (nextRunSeconds > 0) { + remainingSeconds = nextRunSeconds; + updateTimerText(remainingSeconds); + } else { + // No scheduled auto-run; stop polling until the page is refreshed + clearInterval(watchlistCountdownInterval); + watchlistCountdownInterval = null; + updateTimerText(0); } } catch (error) { console.debug('Error updating watchlist countdown:', error); } } else { // Update the display - const timerElement = document.getElementById('watchlist-next-auto-timer'); - if (timerElement) { - const countdownText = formatCountdownTime(remainingSeconds); - timerElement.textContent = `Next Auto${countdownText ? ': ' + countdownText : ''}`; - } + updateTimerText(remainingSeconds); } }, 1000); // Update every second } @@ -3791,4 +3804,3 @@ function cleanupSyncPageLogs() { // --- Global Cleanup on Page Unload --- // Note: Automatic wishlist processing now runs server-side and continues even when browser is closed // =============================== - diff --git a/webui/static/beatport-ui.js b/webui/static/beatport-ui.js index e2c5a774..b692de9e 100644 --- a/webui/static/beatport-ui.js +++ b/webui/static/beatport-ui.js @@ -3638,14 +3638,24 @@ async function loadPlexMusicLibraries() { // Clear existing options selector.innerHTML = ''; - // Add options for each library + // Add options for each library. ``value`` is the canonical + // identifier the backend expects (real libraries: title; + // synthetic "All Libraries" entry: the sentinel string). + // ``title`` stays the human-readable label. data.libraries.forEach(library => { const option = document.createElement('option'); - option.value = library.title; + const optionValue = library.value || library.title; + option.value = optionValue; option.textContent = library.title; - // Mark the currently selected library - if (library.title === data.current || library.title === data.selected) { + // Pre-select match: compare ``value`` against the saved + // DB pref (``data.selected``) AND ``title`` against the + // live-active library name (``data.current``). Covers + // both the sentinel case and the legacy single-library + // case. + if (optionValue === data.selected + || library.title === data.current + || library.title === data.selected) { option.selected = true; } diff --git a/webui/static/core.js b/webui/static/core.js index 3a23cb53..3852fda7 100644 --- a/webui/static/core.js +++ b/webui/static/core.js @@ -471,7 +471,7 @@ function initializeWebSocket() { function handleServiceStatusUpdate(data) { // Cache for library status card - _lastServiceStatus = data; + _lastStatusPayload = data; if (typeof syncSpotifySettingsAuthState === 'function') { syncSpotifySettingsAuthState(data?.spotify || null); @@ -484,11 +484,11 @@ function handleServiceStatusUpdate(data) { } // Same logic as fetchAndUpdateServiceStatus response handler - updateServiceStatus('spotify', data.spotify); + updateServiceStatus('metadata-source', data.metadata_source, data.spotify); updateServiceStatus('media-server', data.media_server); updateServiceStatus('soulseek', data.soulseek); - updateSidebarServiceStatus('spotify', data.spotify); + updateSidebarServiceStatus('metadata-source', data.metadata_source, data.spotify); updateSidebarServiceStatus('media-server', data.media_server); updateSidebarServiceStatus('soulseek', data.soulseek); @@ -881,8 +881,12 @@ const API = { } }; -// Track last service status for library card (used by websocket handler in core + artists) -let _lastServiceStatus = null; +// Track the last `/status` payload (shared service snapshot used across the UI) +let _lastStatusPayload = null; let _isSoulsyncStandalone = false; // Global flag: true when no media server (sync buttons hidden) +function getActiveMetadataSource() { + return _lastStatusPayload?.metadata_source?.source || 'spotify'; +} + // =============================== diff --git a/webui/static/discover.js b/webui/static/discover.js index eb50d2ba..7671d015 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -33,7 +33,6 @@ async function loadDiscoverPage() { loadDiscoverHero(), loadYourArtists(), loadYourAlbums(), - loadSpotifyLibrarySection(), loadDiscoverRecentReleases(), loadSeasonalContent(), // Seasonal discovery loadPersonalizedRecentlyAdded(), // NEW: Recently added from library @@ -1061,17 +1060,33 @@ async function openYourAlbumDownload(index) { if (!album) { showToast('Album data not found', 'error'); return; } showLoadingOverlay(`Loading tracks for ${album.album_name}...`); try { - // Prefer Spotify ID, fall back to Deezer, then search by name + // Per-source dispatch: open with whichever source has an ID for + // this album. For pure-Discogs collection items (no Spotify/ + // Deezer match), dispatch goes straight to Discogs so the + // modal opens with Discogs context (vinyl/CD release detail, + // tracklist from Discogs). For Spotify saved albums (no + // discogs id), goes to Spotify. For multi-source albums + // (album exists in BOTH Spotify saved and Discogs collection, + // rare), tries streaming sources first since they have + // tracklists with proper IDs ready for download. let albumData = null; const nameParams = new URLSearchParams({ name: album.album_name || '', artist: album.artist_name || '' }); - if (album.spotify_album_id) { - const r = await fetch(`/api/discover/album/spotify/${album.spotify_album_id}?${nameParams}`); - if (r.ok) albumData = await r.json(); - } - if (!albumData && album.deezer_album_id) { - const r = await fetch(`/api/discover/album/deezer/${album.deezer_album_id}?${nameParams}`); - if (r.ok) albumData = await r.json(); + const discogsId = album.discogs_release_id || album.discogs_id; + + const trySources = []; + if (album.spotify_album_id) trySources.push(['spotify', album.spotify_album_id]); + if (album.deezer_album_id) trySources.push(['deezer', album.deezer_album_id]); + if (discogsId) trySources.push(['discogs', discogsId]); + + for (const [src, id] of trySources) { + const r = await fetch(`/api/discover/album/${src}/${id}?${nameParams}`); + if (r.ok) { + albumData = await r.json(); + if (albumData && albumData.tracks && albumData.tracks.length > 0) break; + albumData = null; // empty payload — try next + } } + if (!albumData) { // Last resort — search by name const r = await fetch(`/api/discover/album/spotify/search?${nameParams}`); @@ -1157,6 +1172,7 @@ async function openYourAlbumsSourcesModal() { { id: 'spotify', label: 'Spotify', icon: '\uD83C\uDFB5' }, { id: 'tidal', label: 'Tidal', icon: '\uD83C\uDF0A' }, { id: 'deezer', label: 'Deezer', icon: '\uD83C\uDFB6' }, + { id: 'discogs', label: 'Discogs', icon: '\uD83D\uDCBF' }, ]; const state = {}; sourceInfo.forEach(s => { state[s.id] = enabled.includes(s.id); }); @@ -1197,14 +1213,36 @@ async function openYourAlbumsSourcesModal() { window._yaaSourcesState = state; } +// Source-id → human label + setup hint shown when user tries to enable +// a disconnected source. Without this, the toggle silently bailed and +// users saw no feedback — just a non-responsive switch. +const _YAA_DISCONNECTED_HINTS = { + spotify: 'Spotify not connected — log in at Settings → Connections first', + tidal: 'Tidal not connected — set up Tidal in Settings → Connections first', + deezer: 'Deezer not connected — log in or set ARL token at Settings → Connections first', + discogs: 'Discogs not connected — paste your personal access token at Settings → Connections first', +}; + +function _yaaShowDisconnectedHint(id) { + const msg = _YAA_DISCONNECTED_HINTS[id] + || `${id} not connected — set it up in Settings → Connections first`; + if (typeof showToast === 'function') showToast(msg, 'warning'); +} + function _yaaSourceRowClick(id) { const row = document.querySelector(`.ya-source-row[data-yaa-source="${id}"]`); - if (row && row.classList.contains('disconnected')) return; + if (row && row.classList.contains('disconnected')) { + _yaaShowDisconnectedHint(id); + return; + } _yaaSourceToggle(id); } function _yaaSourceToggle(id) { const row = document.querySelector(`.ya-source-row[data-yaa-source="${id}"]`); - if (row && row.classList.contains('disconnected')) return; + if (row && row.classList.contains('disconnected')) { + _yaaShowDisconnectedHint(id); + return; + } window._yaaSourcesState[id] = !window._yaaSourcesState[id]; const btn = document.getElementById(`yaa-toggle-${id}`); if (btn) btn.classList.toggle('on', window._yaaSourcesState[id]); @@ -1221,7 +1259,7 @@ async function _yaaSourcesSave() { if (resp.ok) { document.getElementById('ya-albums-sources-modal-overlay')?.remove(); showToast('Sources saved — refresh to apply', 'success'); - const sourceNames = { spotify: 'Spotify', tidal: 'Tidal', deezer: 'Deezer' }; + const sourceNames = { spotify: 'Spotify', tidal: 'Tidal', deezer: 'Deezer', discogs: 'Discogs' }; const subtitle = document.getElementById('your-albums-subtitle'); if (subtitle) { const names = enabledArr.map(s => sourceNames[s] || s).join(' and '); @@ -1292,341 +1330,6 @@ async function downloadMissingYourAlbums() { } } -// =============================== -// SPOTIFY LIBRARY SECTION -// =============================== - -let spotifyLibraryAlbums = []; -let spotifyLibraryPage = 0; -let spotifyLibraryTotal = 0; -const SPOTIFY_LIBRARY_PAGE_SIZE = 48; -let _spotifyLibrarySearchTimeout = null; - -function debouncedSpotifyLibrarySearch() { - clearTimeout(_spotifyLibrarySearchTimeout); - _spotifyLibrarySearchTimeout = setTimeout(() => { - spotifyLibraryPage = 0; - loadSpotifyLibraryAlbums(); - }, 400); -} - -async function loadSpotifyLibrarySection() { - try { - const section = document.getElementById('spotify-library-section'); - if (!section) return; - - const response = await fetch(`/api/discover/spotify-library?offset=0&limit=${SPOTIFY_LIBRARY_PAGE_SIZE}`); - if (!response.ok) throw new Error('Failed to fetch'); - - const data = await response.json(); - if (!data.success || !data.albums || data.albums.length === 0) { - section.style.display = 'none'; - return; - } - - section.style.display = ''; - spotifyLibraryAlbums = data.albums; - spotifyLibraryTotal = data.total; - spotifyLibraryPage = 0; - - // Update subtitle with stats - const subtitle = document.getElementById('spotify-library-subtitle'); - if (subtitle && data.stats) { - const s = data.stats; - subtitle.textContent = `${s.total} albums \u00B7 ${s.owned} owned \u00B7 ${s.missing} missing`; - } - - // Show download missing button if there are missing albums - const dlBtn = document.getElementById('spotify-library-download-missing-btn'); - if (dlBtn && data.stats && data.stats.missing > 0) { - dlBtn.style.display = ''; - } - - // Show filters - const filters = document.getElementById('spotify-library-filters'); - if (filters) filters.style.display = ''; - - renderSpotifyLibraryGrid(data.albums); - renderSpotifyLibraryPagination(data.total, 0); - - } catch (error) { - console.error('Error loading Spotify library section:', error); - const section = document.getElementById('spotify-library-section'); - if (section) section.style.display = 'none'; - } -} - -async function loadSpotifyLibraryAlbums() { - const grid = document.getElementById('spotify-library-grid'); - if (!grid) return; - - grid.innerHTML = '

Loading...

'; - - try { - const search = (document.getElementById('spotify-library-search')?.value || '').trim(); - const status = document.getElementById('spotify-library-status-filter')?.value || 'all'; - const sort = document.getElementById('spotify-library-sort')?.value || 'date_saved'; - const offset = spotifyLibraryPage * SPOTIFY_LIBRARY_PAGE_SIZE; - - const params = new URLSearchParams({ - offset, limit: SPOTIFY_LIBRARY_PAGE_SIZE, sort, sort_dir: 'desc', status - }); - if (search) params.set('search', search); - - const response = await fetch(`/api/discover/spotify-library?${params}`); - const data = await response.json(); - - if (!data.success) throw new Error(data.error); - - spotifyLibraryAlbums = data.albums; - spotifyLibraryTotal = data.total; - - // Update subtitle - const subtitle = document.getElementById('spotify-library-subtitle'); - if (subtitle && data.stats) { - const s = data.stats; - subtitle.textContent = `${s.total} albums \u00B7 ${s.owned} owned \u00B7 ${s.missing} missing`; - } - - renderSpotifyLibraryGrid(data.albums); - renderSpotifyLibraryPagination(data.total, offset); - - } catch (error) { - console.error('Error loading Spotify library albums:', error); - grid.innerHTML = '

Failed to load albums

'; - } -} - -function renderSpotifyLibraryGrid(albums) { - const grid = document.getElementById('spotify-library-grid'); - if (!grid) return; - - if (!albums || albums.length === 0) { - grid.innerHTML = '

No albums found

'; - return; - } - - let html = ''; - albums.forEach((album, index) => { - const coverUrl = album.image_url || '/static/placeholder-album.png'; - const year = album.release_date ? album.release_date.substring(0, 4) : ''; - const badgeClass = album.in_library ? 'owned' : 'missing'; - const badgeIcon = album.in_library ? '\u2713' : '\u2193'; - const trackInfo = album.total_tracks ? `${album.total_tracks} tracks` : ''; - const meta = [year, trackInfo].filter(Boolean).join(' \u00B7 '); - - html += ` -
-
- ${album.album_name} -
${badgeIcon}
-
-
-

${album.album_name}

-

${album.artist_name}

-

${meta}

-
-
- `; - }); - - grid.innerHTML = html; -} - -function renderSpotifyLibraryPagination(total, offset) { - const container = document.getElementById('spotify-library-pagination'); - if (!container) return; - - if (total <= SPOTIFY_LIBRARY_PAGE_SIZE) { - container.style.display = 'none'; - return; - } - - container.style.display = ''; - const totalPages = Math.ceil(total / SPOTIFY_LIBRARY_PAGE_SIZE); - const currentPage = Math.floor(offset / SPOTIFY_LIBRARY_PAGE_SIZE) + 1; - const showEnd = Math.min(offset + SPOTIFY_LIBRARY_PAGE_SIZE, total); - - container.innerHTML = ` - - ${offset + 1}\u2013${showEnd} of ${total} - - `; -} - -function spotifyLibraryPrevPage() { - if (spotifyLibraryPage > 0) { - spotifyLibraryPage--; - loadSpotifyLibraryAlbums(); - } -} - -function spotifyLibraryNextPage() { - const totalPages = Math.ceil(spotifyLibraryTotal / SPOTIFY_LIBRARY_PAGE_SIZE); - if (spotifyLibraryPage < totalPages - 1) { - spotifyLibraryPage++; - loadSpotifyLibraryAlbums(); - } -} - -async function openSpotifyLibraryAlbumDownload(index) { - const album = spotifyLibraryAlbums[index]; - if (!album) { - showToast('Album data not found', 'error'); - return; - } - - console.log(`\u{1F4E5} Opening download modal for Spotify library album: ${album.album_name}`); - showLoadingOverlay(`Loading tracks for ${album.album_name}...`); - - try { - const _params = new URLSearchParams({ name: album.album_name || '', artist: album.artist_name || '' }); - const response = await fetch(`/api/discover/album/spotify/${album.spotify_album_id}?${_params}`); - if (!response.ok) throw new Error('Failed to fetch album tracks'); - - const albumData = await response.json(); - if (!albumData.tracks || albumData.tracks.length === 0) { - throw new Error('No tracks found in album'); - } - - const spotifyTracks = albumData.tracks.map(track => { - let artists = track.artists || albumData.artists || [{ name: album.artist_name }]; - if (Array.isArray(artists)) { - artists = artists.map(a => a.name || a); - } - return { - id: track.id, - name: track.name, - artists: artists, - album: { - id: albumData.id, - name: albumData.name, - album_type: albumData.album_type || 'album', - total_tracks: albumData.total_tracks || 0, - release_date: albumData.release_date || '', - images: albumData.images || [] - }, - duration_ms: track.duration_ms || 0, - track_number: track.track_number || 0 - }; - }); - - const virtualPlaylistId = `spotify_library_${album.spotify_album_id}`; - const artistContext = { - id: album.artist_id, - name: album.artist_name, - source: 'spotify' - }; - const albumContext = { - id: albumData.id, - name: albumData.name, - album_type: albumData.album_type || 'album', - total_tracks: albumData.total_tracks || 0, - release_date: albumData.release_date || '', - images: albumData.images || [] - }; - - await openDownloadMissingModalForYouTube(virtualPlaylistId, albumData.name, spotifyTracks, artistContext, albumContext); - hideLoadingOverlay(); - - } catch (error) { - console.error('Error opening Spotify library album download:', error); - showToast(`Failed to load album: ${error.message}`, 'error'); - hideLoadingOverlay(); - } -} - -async function refreshSpotifyLibraryCache() { - try { - showToast('Refreshing Spotify library...', 'info'); - const response = await fetch('/api/discover/spotify-library/refresh', { method: 'POST' }); - const data = await response.json(); - if (data.success) { - showToast('Spotify library refresh started — will update shortly', 'success'); - // Reload after a delay to let the sync run - setTimeout(() => loadSpotifyLibrarySection(), 10000); - } else { - showToast(`Error: ${data.error}`, 'error'); - } - } catch (error) { - showToast(`Error: ${error.message}`, 'error'); - } -} - -async function downloadMissingSpotifyLibraryAlbums() { - // Fetch all missing albums (no pagination limit) - try { - const response = await fetch('/api/discover/spotify-library?status=missing&limit=500&offset=0'); - const data = await response.json(); - if (!data.success || !data.albums || data.albums.length === 0) { - showToast('No missing albums to download', 'info'); - return; - } - - const missing = data.albums.filter(a => !a.in_library); - if (missing.length === 0) { - showToast('All albums are already in your library!', 'success'); - return; - } - - if (!confirm(`Download ${missing.length} missing album${missing.length > 1 ? 's' : ''} from your Spotify library?`)) { - return; - } - - showToast(`Starting download for ${missing.length} albums...`, 'info'); - - // Download one at a time to avoid overwhelming the system - for (let i = 0; i < missing.length; i++) { - const album = missing[i]; - try { - showToast(`Queuing ${i + 1}/${missing.length}: ${album.album_name}`, 'info'); - - const _params = new URLSearchParams({ name: album.album_name || '', artist: album.artist_name || '' }); - const response = await fetch(`/api/discover/album/spotify/${album.spotify_album_id}?${_params}`); - if (!response.ok) continue; - - const albumData = await response.json(); - if (!albumData.tracks || albumData.tracks.length === 0) continue; - - const spotifyTracks = albumData.tracks.map(track => { - let artists = track.artists || albumData.artists || [{ name: album.artist_name }]; - if (Array.isArray(artists)) artists = artists.map(a => a.name || a); - return { - id: track.id, - name: track.name, - artists: artists, - album: { - id: albumData.id, - name: albumData.name, - album_type: albumData.album_type || 'album', - total_tracks: albumData.total_tracks || 0, - release_date: albumData.release_date || '', - images: albumData.images || [] - }, - duration_ms: track.duration_ms || 0, - track_number: track.track_number || 0 - }; - }); - - const virtualPlaylistId = `spotify_library_${album.spotify_album_id}`; - await openDownloadMissingModalForYouTube(virtualPlaylistId, albumData.name, spotifyTracks, { - id: album.artist_id, name: album.artist_name, source: 'spotify' - }, { - id: albumData.id, name: albumData.name, album_type: albumData.album_type || 'album', - total_tracks: albumData.total_tracks || 0, release_date: albumData.release_date || '', - images: albumData.images || [] - }); - - } catch (err) { - console.error(`Error downloading album ${album.album_name}:`, err); - } - } - - } catch (error) { - console.error('Error downloading missing Spotify library albums:', error); - showToast(`Error: ${error.message}`, 'error'); - } -} async function loadDiscoverReleaseRadar() { try { diff --git a/webui/static/downloads.js b/webui/static/downloads.js index 1720038b..8db1bc0e 100644 --- a/webui/static/downloads.js +++ b/webui/static/downloads.js @@ -3305,127 +3305,133 @@ function processModalStatusUpdate(playlistId, data) { // Note: Auto-show logic removed - wishlist modal visibility managed by user interaction only if (data.phase === 'cancelled') { - process.status = 'cancelled'; + if (process.status !== 'cancelled') { + process.status = 'cancelled'; - // Reset YouTube playlist phase to 'discovered' if this is a YouTube playlist on cancel - if (playlistId.startsWith('youtube_')) { - const urlHash = playlistId.replace('youtube_', ''); - updateYouTubeCardPhase(urlHash, 'discovered'); - if (urlHash.startsWith('mirrored_')) { - updateMirroredCardPhase(urlHash, 'discovered'); + // Reset YouTube playlist phase to 'discovered' if this is a YouTube playlist on cancel + if (playlistId.startsWith('youtube_')) { + const urlHash = playlistId.replace('youtube_', ''); + updateYouTubeCardPhase(urlHash, 'discovered'); + if (urlHash.startsWith('mirrored_')) { + updateMirroredCardPhase(urlHash, 'discovered'); + } } - } - showToast(`Process cancelled for ${process.playlist.name}.`, 'info'); + showToast(`Process cancelled for ${process.playlist.name}.`, 'info'); + } } else if (data.phase === 'error') { - process.status = 'complete'; // Treat as complete to allow cleanup - updatePlaylistCardUI(playlistId); // Update card to show ready for review + if (process.status !== 'complete') { + process.status = 'complete'; + updatePlaylistCardUI(playlistId); // Update card to show ready for review - // Reset YouTube playlist phase to 'discovered' if this is a YouTube playlist on error - if (playlistId.startsWith('youtube_')) { - const urlHash = playlistId.replace('youtube_', ''); - updateYouTubeCardPhase(urlHash, 'discovered'); - if (urlHash.startsWith('mirrored_')) { - updateMirroredCardPhase(urlHash, 'discovered'); + // Reset YouTube playlist phase to 'discovered' if this is a YouTube playlist on error + if (playlistId.startsWith('youtube_')) { + const urlHash = playlistId.replace('youtube_', ''); + updateYouTubeCardPhase(urlHash, 'discovered'); + if (urlHash.startsWith('mirrored_')) { + updateMirroredCardPhase(urlHash, 'discovered'); + } } - } - showToast(`Process for ${process.playlist.name} failed!`, 'error'); + showToast(`Process for ${process.playlist.name} failed!`, 'error'); + } } else { - process.status = 'complete'; - updatePlaylistCardUI(playlistId); // Update card to show ready for review + if (process.status !== 'complete') { + process.status = 'complete'; + updatePlaylistCardUI(playlistId); // Update card to show ready for review - // Update YouTube playlist phase to 'download_complete' if this is a YouTube playlist - if (playlistId.startsWith('youtube_')) { - const urlHash = playlistId.replace('youtube_', ''); - updateYouTubeCardPhase(urlHash, 'download_complete'); - if (urlHash.startsWith('mirrored_')) { - updateMirroredCardPhase(urlHash, 'download_complete'); - } - } - - // Update Tidal playlist phase to 'download_complete' if this is a Tidal playlist - if (playlistId.startsWith('tidal_')) { - const tidalPlaylistId = playlistId.replace('tidal_', ''); - if (tidalPlaylistStates[tidalPlaylistId]) { - tidalPlaylistStates[tidalPlaylistId].phase = 'download_complete'; - // Store the download process ID for potential modal rehydration - tidalPlaylistStates[tidalPlaylistId].download_process_id = process.batchId; - updateTidalCardPhase(tidalPlaylistId, 'download_complete'); - console.log(`✅ [Status Complete] Updated Tidal playlist ${tidalPlaylistId} to download_complete phase`); - } - } - - // Update Beatport chart phase to 'download_complete' if this is a Beatport chart - if (playlistId.startsWith('beatport_')) { - const urlHash = playlistId.replace('beatport_', ''); - const state = youtubePlaylistStates[urlHash]; - - if (state && state.is_beatport_playlist) { - const chartHash = state.beatport_chart_hash || urlHash; - - // Update frontend states - state.phase = 'download_complete'; - state.download_process_id = process.batchId; - if (beatportChartStates[chartHash]) { - beatportChartStates[chartHash].phase = 'download_complete'; + // Update YouTube playlist phase to 'download_complete' if this is a YouTube playlist + if (playlistId.startsWith('youtube_')) { + const urlHash = playlistId.replace('youtube_', ''); + updateYouTubeCardPhase(urlHash, 'download_complete'); + if (urlHash.startsWith('mirrored_')) { + updateMirroredCardPhase(urlHash, 'download_complete'); } + } - // Update card UI - updateBeatportCardPhase(chartHash, 'download_complete'); - - // Update backend state - try { - fetch(`/api/beatport/charts/update-phase/${chartHash}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - phase: 'download_complete', - download_process_id: process.batchId - }) - }); - } catch (error) { - console.warn('⚠️ Error updating backend Beatport phase to download_complete:', error); + // Update Tidal playlist phase to 'download_complete' if this is a Tidal playlist + if (playlistId.startsWith('tidal_')) { + const tidalPlaylistId = playlistId.replace('tidal_', ''); + if (tidalPlaylistStates[tidalPlaylistId]) { + tidalPlaylistStates[tidalPlaylistId].phase = 'download_complete'; + // Store the download process ID for potential modal rehydration + tidalPlaylistStates[tidalPlaylistId].download_process_id = process.batchId; + updateTidalCardPhase(tidalPlaylistId, 'download_complete'); + console.log(`✅ [Status Complete] Updated Tidal playlist ${tidalPlaylistId} to download_complete phase`); } - - console.log(`✅ [Status Complete] Updated Beatport chart ${chartHash} to download_complete phase`); } - } - // Handle background wishlist processing completion specially - if (isBackgroundWishlist) { - console.log(`🎉 Background wishlist processing complete: ${completedCount} downloaded, ${notFoundCount} not found, ${failedOrCancelledCount} failed`); + // Update Beatport chart phase to 'download_complete' if this is a Beatport chart + if (playlistId.startsWith('beatport_')) { + const urlHash = playlistId.replace('beatport_', ''); + const state = youtubePlaylistStates[urlHash]; - // Reset modal to idle state to prevent "complete" phase disruption - setTimeout(() => { - resetWishlistModalToIdleState(); - // Server-side auto-processing will handle next cycle automatically - }, 500); + if (state && state.is_beatport_playlist) { + const chartHash = state.beatport_chart_hash || urlHash; - return; // Skip normal completion handling - } + // Update frontend states + state.phase = 'download_complete'; + state.download_process_id = process.batchId; + if (beatportChartStates[chartHash]) { + beatportChartStates[chartHash].phase = 'download_complete'; + } - // Show completion summary with wishlist stats (matching sync.py behavior) - let completionMessage = `Process complete for ${process.playlist.name}!`; - let messageType = 'success'; + // Update card UI + updateBeatportCardPhase(chartHash, 'download_complete'); - // Check for wishlist summary from backend (added when failed/cancelled tracks are processed) - if (data.wishlist_summary) { - const summary = data.wishlist_summary; - let summaryParts = [`Downloaded: ${completedCount}`]; - if (notFoundCount > 0) summaryParts.push(`Not Found: ${notFoundCount}`); - if (failedOrCancelledCount > 0) summaryParts.push(`Failed: ${failedOrCancelledCount}`); - completionMessage = `Download process complete! ${summaryParts.join(', ')}.`; + // Update backend state + try { + fetch(`/api/beatport/charts/update-phase/${chartHash}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + phase: 'download_complete', + download_process_id: process.batchId + }) + }); + } catch (error) { + console.warn('⚠️ Error updating backend Beatport phase to download_complete:', error); + } - if (summary.tracks_added > 0) { - completionMessage += ` Added ${summary.tracks_added} failed track${summary.tracks_added !== 1 ? 's' : ''} to wishlist for automatic retry.`; - } else if (summary.total_failed > 0) { - completionMessage += ` ${summary.total_failed} track${summary.total_failed !== 1 ? 's' : ''} could not be added to wishlist.`; - messageType = 'warning'; + console.log(`✅ [Status Complete] Updated Beatport chart ${chartHash} to download_complete phase`); + } } - } - showToast(completionMessage, messageType); + // Handle background wishlist processing completion specially + if (isBackgroundWishlist) { + console.log(`🎉 Background wishlist processing complete: ${completedCount} downloaded, ${notFoundCount} not found, ${failedOrCancelledCount} failed`); + + // Reset modal to idle state to prevent "complete" phase disruption + setTimeout(() => { + resetWishlistModalToIdleState(); + // Server-side auto-processing will handle next cycle automatically + }, 500); + + return; // Skip normal completion handling + } + + // Show completion summary with wishlist stats (matching sync.py behavior) + let completionMessage = `Process complete for ${process.playlist.name}!`; + let messageType = 'success'; + + // Check for wishlist summary from backend (added when failed/cancelled tracks are processed) + if (data.wishlist_summary) { + const summary = data.wishlist_summary; + let summaryParts = [`Downloaded: ${completedCount}`]; + if (notFoundCount > 0) summaryParts.push(`Not Found: ${notFoundCount}`); + if (failedOrCancelledCount > 0) summaryParts.push(`Failed: ${failedOrCancelledCount}`); + completionMessage = `Download process complete! ${summaryParts.join(', ')}.`; + + if (summary.tracks_added > 0) { + completionMessage += ` Added ${summary.tracks_added} failed track${summary.tracks_added !== 1 ? 's' : ''} to wishlist for automatic retry.`; + } else if (summary.total_failed > 0) { + completionMessage += ` ${summary.total_failed} track${summary.total_failed !== 1 ? 's' : ''} could not be added to wishlist.`; + messageType = 'warning'; + } + } + + showToast(completionMessage, messageType); + } } document.getElementById(`cancel-all-btn-${playlistId}`).style.display = 'none'; diff --git a/webui/static/enrichment.js b/webui/static/enrichment.js index 3a361baf..23c44935 100644 --- a/webui/static/enrichment.js +++ b/webui/static/enrichment.js @@ -8,7 +8,7 @@ async function updateMusicBrainzStatus() { if (socketConnected) return; // WebSocket handles this if (document.hidden) return; // Skip polling when tab is not visible try { - const response = await fetch('/api/musicbrainz/status'); + const response = await fetch('/api/enrichment/musicbrainz/status'); if (!response.ok) { console.warn('MusicBrainz status endpoint unavailable'); return; } const data = await response.json(); updateMusicBrainzStatusFromData(data); @@ -94,7 +94,7 @@ async function toggleMusicBrainzEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/musicbrainz/pause' : '/api/musicbrainz/resume'; + const endpoint = isRunning ? '/api/enrichment/musicbrainz/pause' : '/api/enrichment/musicbrainz/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -146,7 +146,7 @@ async function updateAudioDBStatus() { if (socketConnected) return; // WebSocket handles this if (document.hidden) return; // Skip polling when tab is not visible try { - const response = await fetch('/api/audiodb/status'); + const response = await fetch('/api/enrichment/audiodb/status'); if (!response.ok) { console.warn('AudioDB status endpoint unavailable'); return; } const data = await response.json(); updateAudioDBStatusFromData(data); @@ -257,7 +257,7 @@ async function toggleDiscogsEnrichment() { const button = document.getElementById('discogs-button'); if (!button) return; const isPaused = button.classList.contains('paused') || button.classList.contains('complete'); - const endpoint = isPaused ? '/api/discogs/resume' : '/api/discogs/pause'; + const endpoint = isPaused ? '/api/enrichment/discogs/resume' : '/api/enrichment/discogs/pause'; const response = await fetch(endpoint, { method: 'POST' }); if (response.ok) { showToast(isPaused ? 'Discogs enrichment resumed' : 'Discogs enrichment paused', 'info'); @@ -276,7 +276,7 @@ async function toggleAudioDBEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/audiodb/pause' : '/api/audiodb/resume'; + const endpoint = isRunning ? '/api/enrichment/audiodb/pause' : '/api/enrichment/audiodb/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -323,7 +323,7 @@ async function updateDeezerStatus() { if (socketConnected) return; // WebSocket handles this if (document.hidden) return; // Skip polling when tab is not visible try { - const response = await fetch('/api/deezer/status'); + const response = await fetch('/api/enrichment/deezer/status'); if (!response.ok) { console.warn('Deezer status endpoint unavailable'); return; } const data = await response.json(); updateDeezerStatusFromData(data); @@ -395,7 +395,7 @@ async function toggleDeezerEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/deezer/pause' : '/api/deezer/resume'; + const endpoint = isRunning ? '/api/enrichment/deezer/pause' : '/api/enrichment/deezer/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -442,7 +442,7 @@ async function updateSpotifyEnrichmentStatus() { if (socketConnected) return; // WebSocket handles this if (document.hidden) return; // Skip polling when tab is not visible try { - const response = await fetch('/api/spotify-enrichment/status'); + const response = await fetch('/api/enrichment/spotify/status'); if (!response.ok) { console.warn('Spotify enrichment status endpoint unavailable'); return; } const data = await response.json(); updateSpotifyEnrichmentStatusFromData(data); @@ -548,7 +548,7 @@ async function toggleSpotifyEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/spotify-enrichment/pause' : '/api/spotify-enrichment/resume'; + const endpoint = isRunning ? '/api/enrichment/spotify/pause' : '/api/enrichment/spotify/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -596,7 +596,7 @@ async function updateiTunesEnrichmentStatus() { if (socketConnected) return; // WebSocket handles this if (document.hidden) return; // Skip polling when tab is not visible try { - const response = await fetch('/api/itunes-enrichment/status'); + const response = await fetch('/api/enrichment/itunes/status'); if (!response.ok) { console.warn('iTunes enrichment status endpoint unavailable'); return; } const data = await response.json(); updateiTunesEnrichmentStatusFromData(data); @@ -672,7 +672,7 @@ async function toggleiTunesEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/itunes-enrichment/pause' : '/api/itunes-enrichment/resume'; + const endpoint = isRunning ? '/api/enrichment/itunes/pause' : '/api/enrichment/itunes/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -715,7 +715,7 @@ async function updateLastFMEnrichmentStatus() { if (socketConnected) return; if (document.hidden) return; try { - const response = await fetch('/api/lastfm-enrichment/status'); + const response = await fetch('/api/enrichment/lastfm/status'); if (!response.ok) { console.warn('Last.fm status endpoint unavailable'); return; } const data = await response.json(); updateLastFMEnrichmentStatusFromData(data); @@ -800,7 +800,7 @@ async function toggleLastFMEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/lastfm-enrichment/pause' : '/api/lastfm-enrichment/resume'; + const endpoint = isRunning ? '/api/enrichment/lastfm/pause' : '/api/enrichment/lastfm/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -842,7 +842,7 @@ async function updateGeniusEnrichmentStatus() { if (socketConnected) return; if (document.hidden) return; try { - const response = await fetch('/api/genius-enrichment/status'); + const response = await fetch('/api/enrichment/genius/status'); if (!response.ok) { console.warn('Genius status endpoint unavailable'); return; } const data = await response.json(); updateGeniusEnrichmentStatusFromData(data); @@ -921,7 +921,7 @@ async function toggleGeniusEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/genius-enrichment/pause' : '/api/genius-enrichment/resume'; + const endpoint = isRunning ? '/api/enrichment/genius/pause' : '/api/enrichment/genius/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -963,7 +963,7 @@ async function updateTidalEnrichmentStatus() { if (socketConnected) return; if (document.hidden) return; try { - const response = await fetch('/api/tidal-enrichment/status'); + const response = await fetch('/api/enrichment/tidal/status'); if (!response.ok) { console.warn('Tidal status endpoint unavailable'); return; } const data = await response.json(); updateTidalEnrichmentStatusFromData(data); @@ -1046,7 +1046,7 @@ async function toggleTidalEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/tidal-enrichment/pause' : '/api/tidal-enrichment/resume'; + const endpoint = isRunning ? '/api/enrichment/tidal/pause' : '/api/enrichment/tidal/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -1088,7 +1088,7 @@ async function updateQobuzEnrichmentStatus() { if (socketConnected) return; if (document.hidden) return; try { - const response = await fetch('/api/qobuz-enrichment/status'); + const response = await fetch('/api/enrichment/qobuz/status'); if (!response.ok) { console.warn('Qobuz status endpoint unavailable'); return; } const data = await response.json(); updateQobuzEnrichmentStatusFromData(data); @@ -1171,7 +1171,7 @@ async function toggleQobuzEnrichment() { if (!button) return; const isRunning = button.classList.contains('active'); - const endpoint = isRunning ? '/api/qobuz-enrichment/pause' : '/api/qobuz-enrichment/resume'; + const endpoint = isRunning ? '/api/enrichment/qobuz/pause' : '/api/enrichment/qobuz/resume'; const response = await fetch(endpoint, { method: 'POST' }); if (!response.ok) { @@ -1425,6 +1425,7 @@ let _repairCurrentTab = 'jobs'; let _repairFindingsPage = 0; let _repairSelectedFindings = new Set(); let _repairFindingsTotal = 0; +let _repairFindingsAutoSwitched = false; // Set after auto-switching to "All Status" so we don't loop const REPAIR_FINDINGS_PAGE_SIZE = 30; let _repairJobsCache = {}; // Cache job data for help modal @@ -1554,11 +1555,25 @@ async function loadRepairJobs() { flowParts.push('Auto-fix'); } } - // Show pending findings count - const findingsCount = job.last_run ? (job.last_run.findings_created || 0) : 0; - if (findingsCount > 0) { + // Badge: prefer the CURRENT pending count from the API + // (matches what the Findings tab actually shows); fall + // back to the historical ``findings_created`` from the + // last run when pending = 0 but the last scan did find + // something — that way users still see a hint that a scan + // happened recently. Without this, a scan that finds 372 + // duplicates and then has them all bulk-fixed shows + // "372 findings" on the badge while the Findings tab + // Pending filter is empty, which reads as a bug. + const pendingCount = job.pending_findings_count || 0; + const lastScanCount = job.last_run ? (job.last_run.findings_created || 0) : 0; + if (pendingCount > 0) { flowParts.push('→'); - flowParts.push(`${findingsCount} finding${findingsCount !== 1 ? 's' : ''}`); + flowParts.push(`${pendingCount} pending`); + } else if (lastScanCount > 0) { + // Show historical count with a clear "found in last scan" + // qualifier so users don't expect them on the Pending tab. + flowParts.push('→'); + flowParts.push(`${lastScanCount} found in last scan`); } // Build meta parts @@ -2459,6 +2474,38 @@ async function loadRepairFindings() { if (selectAllCb) { selectAllCb.checked = false; selectAllCb.indeterminate = false; } if (items.length === 0) { + // If the user is on the default "pending" filter and there are + // ZERO pending rows but other statuses (dismissed/resolved) do + // have rows, auto-switch the filter to "All Status" so the user + // sees the carry-over findings instead of an empty pane. Common + // case: scanner re-found same issues that were dismissed + // previously — dedup-skip means no new pending row, so the + // default-filtered tab looks empty even though the badge count + // referenced existing dismissed rows. + if (statusFilter && statusFilter.value === 'pending' && !_repairFindingsAutoSwitched) { + try { + const countsResp = await fetch('/api/repair/findings/counts'); + if (countsResp.ok) { + const counts = await countsResp.json(); + const otherTotal = (counts.resolved || 0) + (counts.dismissed || 0) + (counts.auto_fixed || 0); + if (otherTotal > 0) { + _repairFindingsAutoSwitched = true; + statusFilter.value = ''; + _repairFindingsPage = 0; + await loadRepairFindings(); + const list = document.getElementById('repair-findings-list'); + if (list && !list.querySelector('.repair-auto-switch-notice')) { + const notice = document.createElement('div'); + notice.className = 'repair-auto-switch-notice'; + notice.style.cssText = 'background:rgba(96,165,250,0.08);border:1px solid rgba(96,165,250,0.25);border-radius:8px;padding:10px 14px;margin-bottom:12px;font-size:13px;color:#cbd5e1;'; + notice.innerHTML = `No pending findings, but ${otherTotal.toLocaleString()} carry-over (resolved/dismissed/auto-fixed). Showing All Status — change the filter above to switch back.`; + list.parentNode.insertBefore(notice, list); + } + return; + } + } + } catch (e) { /* fall through to empty state */ } + } container.innerHTML = `
✓
All Clear
diff --git a/webui/static/helper.js b/webui/static/helper.js index b58876e1..255d5ed0 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -173,7 +173,7 @@ const HELPER_CONTENT = { title: 'Support & Community', description: 'Links to the SoulSync community Discord, GitHub issues for bug reports, and documentation resources.', }, - '#spotify-indicator': { + '#metadata-source-indicator': { title: 'Metadata Source', description: 'Connection status of your primary metadata source. This service provides artist, album, and track information for searches, enrichment, and discovery.', tips: [ @@ -236,7 +236,7 @@ const HELPER_CONTENT = { // ─── DASHBOARD: SERVICE CARDS ─────────────────────────────────── - '#spotify-service-card': { + '#metadata-source-service-card': { title: 'Metadata Source Status', description: 'Detailed connection info for your active metadata source. Shows connection state, response latency, and allows manual connection testing.', tips: [ @@ -1230,17 +1230,6 @@ const HELPER_CONTENT = { description: 'Click to browse tracks in this genre from your discovery pool.', }, - // Spotify Library - '#spotify-library-section': { - title: 'Your Spotify Library', - description: 'Albums saved in your Spotify account. Browse, search, and download albums you\'ve saved on Spotify but don\'t have locally.', - tips: [ - 'Search and filter by status (All/Missing/Owned)', - 'Sort by date saved, artist, album name, or release date', - '"Download Missing" downloads all albums not in your library' - ], - }, - // Playlist Sync/Download buttons (generic — matches all discover playlist sections) '.discover-section-actions .action-button.primary': { title: 'Sync to Media Server', @@ -2375,7 +2364,7 @@ const HELPER_TOURS = { { page: 'dashboard', selector: '#wishlist-button', title: 'Wishlist', description: 'Tracks queued for download. Failed downloads, watchlist discoveries, and manual additions all land here for retry.' }, // Service cards - { page: 'dashboard', selector: '#spotify-service-card', title: 'Metadata Source', description: 'Shows your metadata source connection (Spotify, iTunes, or Deezer). This determines where album, artist, and track info comes from. Click "Test Connection" to verify.' }, + { page: 'dashboard', selector: '#metadata-source-service-card', title: 'Metadata Source', description: 'Shows your metadata source connection (Spotify, iTunes, or Deezer). This determines where album, artist, and track info comes from. Click "Test Connection" to verify.' }, { page: 'dashboard', selector: '#media-server-service-card', title: 'Media Server', description: 'Your media server (Plex, Jellyfin, or Navidrome). This is where your music library lives. SoulSync reads your collection and sends downloads here.' }, { page: 'dashboard', selector: '#soulseek-service-card', title: 'Download Source', description: 'Your primary download source status. In hybrid mode, shows the first source in your priority chain.' }, @@ -2485,7 +2474,6 @@ const HELPER_TOURS = { { page: 'discover', selector: '#discover-hero-view-all', title: 'View All Recommendations', description: 'Opens a modal with all recommended artists at once. "Watch All" adds every recommended artist to your watchlist in one click.' }, // Content sections (top to bottom) - { page: 'discover', selector: '#spotify-library-section', title: 'Your Spotify Library', description: 'If Spotify is connected, this shows all your saved albums. Filter by Missing/Owned, sort by date, and click "Download Missing" to grab everything you don\'t have yet. Only visible with Spotify connected.' }, { page: 'discover', selector: '#recent-releases-carousel', title: 'Recent Releases', description: 'New music from artists in your watchlist. Album cards show cover art — click any to open the download modal. Updates automatically when watchlist scans find new releases.' }, { page: 'discover', selector: '#seasonal-albums-section', title: 'Seasonal Content', description: 'Season-aware sections that appear automatically — Christmas albums in December, summer vibes in July. Includes curated albums and a Seasonal Mix playlist you can sync to your server.' }, @@ -2974,13 +2962,13 @@ async function _checkSetupStatus() { const completion = _getSetupCompletion(); const results = { ...completion }; - // ── /status — checks services (spotify, media_server, soulseek) ───── + // ── /status — checks metadata_source, media_server, soulseek ──────── try { const resp = await fetch('/status'); if (resp.ok) { const data = await resp.json(); // Metadata source is available when status reports a source. - if (data.spotify?.source) { + if (data.metadata_source?.source) { results['metadata-source'] = results['metadata-source'] || Date.now(); _markSetupComplete('metadata-source'); } @@ -3441,6 +3429,55 @@ function closeHelperSearch() { // projects that span multiple commits before shipping. Strip the flag at // release time and add a real `date:` line at the top of the version block. const WHATS_NEW = { + '2.4.2': [ + // --- May 7, 2026 — patch release --- + { date: 'May 7, 2026 — 2.4.2 release' }, + { title: 'Artist Top Tracks: Per-Row + Bulk Download', desc: 'github issue #513 (s66jones): wanted a way to grab an artist\'s top X popular songs without pulling the full discography (the zotify workflow). artist detail page already had a "popular on last.fm" sidebar, but it was display-only — play button per row, no download. now when your primary metadata source is spotify or deezer, that sidebar pulls top tracks via the source\'s native popularity endpoint (spotify `artist_top_tracks` returns 10 per market, deezer `/artist/{id}/top` supports up to 100), each row gets a wishlist-add button on hover, and a "download all" footer button opens the existing wishlist modal with all top tracks pre-loaded. files land in their REAL album folders on disk (not a fake "top tracks" folder) because each track carries its actual album metadata. itunes / discogs / musicbrainz primary still falls back to the existing last.fm playcount display (no popularity ranking on those sources). 10 new tests pin the spotify + deezer client method behavior (auth gate, limit clamping, malformed response handling, spotify-compatible shape conversion).', page: 'library' }, + { title: 'Fix: AcoustID Verification Let Instrumentals Pass As Vocal Tracks', desc: 'discord report (corruption [BWC]): downloads coming through as instrumental versions when the user expected the vocal cut. slipped past acoustid verification because the title-similarity normalizer strips parentheticals and version-suffix tags ("(Instrumental)", "- Live", etc) so legit name variations don\'t false-fail the comparison. side effect: "in my feelings" and "in my feelings (instrumental)" both normalize to "in my feelings", title sim is 1.0, file passes verification despite being the wrong cut. fix: detect the version label on each side BEFORE normalization runs — if expected and matched disagree (one is original, the other is instrumental / live / acoustic / remix / etc), reject as version mismatch. reuses `MusicMatchingEngine.detect_version_type` so post-download verification uses the same patterns the pre-download soulseek matcher already applies (no duplicated regex tables). also gates the secondary fallback scan, so a wrong-version variant in the same fingerprint cluster can\'t win the loop after the best match is rejected. 6 new tests pin the four direction cases (instrumental returned for vocal request → fail, vocal returned for instrumental request → fail, live vs acoustic → fail, matching versions on both sides → pass) plus the original-to-original happy path and the secondary-scan gate.', page: 'downloads' }, + { title: 'Fix: Search Picker Defaulted to Spotify on Non-Admin Profiles', desc: 'github issue #515 (jaruca): admin sets primary metadata source to deezer / itunes / discogs, but every non-admin profile saw spotify as the active source on the search page and global search popover, requiring manual click each time. cause: `shared-helpers.js` resolved the active source by fetching `/api/settings` — that endpoint is `@admin_only` because it returns full config including credentials, so non-admin profiles got 403 and silently fell back to the hardcoded `spotify` default. fix: read from `/status` instead, which is public and already returns `metadata_source` for the dashboard. one-line scope change, behavior preserved for admins (same value, different endpoint), non-admins now see the real configured source.', page: 'search' }, + { title: 'Internal: Stop Swallowing Exceptions Silently', desc: 'github issue #369 (johnbaumb): the codebase had ~300 `except Exception: pass` blocks — and another ~30 bare `except: pass` ones — across web_server.py, every metadata client, every download/import worker, the repair jobs, and most service modules. when one of those paths failed at runtime, the failure was completely invisible: no log line, no telemetry, nothing. you\'d see "downloads stopped working after a few hours" or "enrichment never finishes" and there was nothing to grep for in app.log because the exception had been thrown straight into the void. swept all of them. converted to `except Exception as e: logger.debug(": %s", e)` so failures land in the log with enough context to grep. bare `except:` cases (which also swallow KeyboardInterrupt and SystemExit — actively wrong) got upgraded to `except Exception:` first so ctrl-c works correctly. ~14 cleanup-path sites (atexit handlers, finally-block conn.close calls) were intentionally left silent with explicit `# noqa: S110` comments — logging during shutdown can itself crash because file handles get torn down before the handler fires. and added ruff S110 to the lint config so this pattern fails CI going forward — drift fails at PR review instead of at runtime against a wedged worker thread. zero behavior change to any happy path; just made the failure paths inspectable. test suite (2188 tests) green throughout the sweep.' }, + { title: 'Fix: Repair Job Card "X Findings" Badge Was Misleading After Bulk-Fix', desc: 'discord report: duplicate detector card said "372 findings" and cover art filler said "60 findings", but clicking the findings tab pending filter showed 0 — read like a bug ("findings aren\'t being created"). actual cause: job-card badge displayed `last_run.findings_created` (historical "found in last scan") which doesn\'t reflect current state when those findings have since been bulk-fixed and moved to status="resolved". fix: api response now includes `pending_findings_count` per job (current pending count from a single sql aggregation). badge now shows "X pending" when pending count > 0 (urgent red color), or "X found in last scan" with a muted grey color when pending = 0 but the last scan did surface something. user can tell at a glance whether something needs review vs whether it\'s a historical reminder. 3 new tests pin the per-job pending count helper.', page: 'stats' }, + { title: 'Fix: Downloads Stop After 2-3 Hours (slskd HTTP Timeout)', desc: 'github issue #499 (bafoed): big initial sync of spotify playlists worked for 2-3 hours then downloads silently stopped. 3 active tasks stuck in "searching" state, replaced every ~10 min by different ones, but slskd ui showed no actual searches happening. only fix was restarting the soulsync container — which would buy another 2-3 hours. root cause: `core/soulseek_client.py` constructed `aiohttp.ClientSession()` with no timeout at four sites. when slskd hung on a request (overloaded, transient network blip, internal stall), the http call blocked indefinitely — and the worker thread blocked with it. download executor only has `max_workers=3` for download workers. once 3 worker threads were wedged on hung calls, no further downloads could start. batch-level "stuck detection" (10-min) was correctly marking tasks `not_found` and trying to start replacements, but the executor pool was exhausted — replacements queued forever. fix: bounded `aiohttp.ClientTimeout` (total 120s, connect 15s, sock_read 60s) on every slskd `ClientSession` construction. legitimate metadata calls (search submission, status polls, download enqueue) finish in seconds — slskd doesn\'t stream files through these requests, so the timeout can\'t kill a real operation. when timeout fires, the request raises `asyncio.TimeoutError` which is now explicitly caught + logged + returns None to the caller (treats as a normal failure, same code path as a 5xx response). worker thread unblocks → executor pool stays healthy → downloads keep flowing. 3 new tests pin the timeout config + the `asyncio.TimeoutError` handler so future drift fails at the test boundary instead of at runtime against a wedged executor.', page: 'downloads' }, + { title: 'Fix: Library Reorganize Job Misclassified Album Tracks As Singles', desc: 'github issue #500 (bafoed): library reorganize repair job moved tracks like `Surf Curse/Surf Curse - Nothing Yet (2017)/01 - Christine F.flac` to single-template paths like `Surf Curse/Surf Curse - Christine F/Surf Curse - Christine F.flac`. root cause: the job used `is_album = (group_size > 1)` where `group_size` was the count of tracks for the same album currently sitting in the transfer folder being scanned — when only one track of an album was in transfer (rest already moved to library, or album tags varied across tracks like "Buds" vs "Buds (Bonus)"), each track became a 1-element group → all routed through single template. fix: rewrote the job to delegate to the per-album planner (`core.library_reorganize.preview_album_reorganize` / `reorganize_queue`) — the same planner the artist-detail "reorganize" modal uses. db-driven: the planner knows the album has multiple tracks regardless of how many sit in the transfer folder, so the album-vs-single classification is structurally correct. apply mode delegates to the existing reorganize queue → file move + post-processing + db update + sidecar handling all flow through one code path. only iterates albums for the ACTIVE media server (matches the artist-detail modal\'s scope) — multi-server users (plex + jellyfin etc) won\'t accidentally have the job touch the inactive server\'s files. albums missing a metadata source id get a single "needs enrichment first" finding instead of n per-track "no source" findings. dropped ~500 loc of tag-reading + transfer-walk + template logic that was duplicated against the per-album path. files in transfer with no db entry are now exclusively the orphan_file_detector\'s domain (clean separation). 12 tests pin the delegation contract.', page: 'library' }, + { title: 'Fix: Enrich Honors Manual Album Matches', desc: 'github issue #501 (tacobell444): if you manually matched an album to a specific source ID via the match-chip UI, then clicked "enrich" on that album, the worker would search by name and overwrite your manual match with whatever the search returned (or revert status to "not_found" if it found nothing). reorganize then read the now-wrong id and moved files to the wrong destination. fix: extracted a shared `core/enrichment/manual_match_honoring.py` helper. every per-source enrichment worker (spotify / itunes / deezer / tidal / qobuz) now reads its stored id column at the top of `_process_*_individual` — if present, it fetches via `client.get_album(stored_id)` directly and refreshes metadata without touching the id. fuzzy name search only runs as fallback for never-matched entities. discogs / audiodb / musicbrainz already had inline stored-id fast paths and are left alone. lastfm / genius are name-based and don\'t store ids. cin-shape lift: same fix in 5 workers gets exactly one helper, per-worker variability (column name, client method, response shape) plugs in via callbacks. 11 new helper tests pin: stored-id fast-path, no-id fallthrough, fetch-failure fallthrough, table/column whitelist, callback contract.', page: 'library' }, + { title: 'Fix: "no such table: hifi_instances" When Adding HiFi Instance', desc: 'github issue #503 (hadshaw21): adding a hifi instance via downloader settings popped up `no such table: hifi_instances` even though the connection test and "check all instances" both worked. root cause: `_initialize_database` runs every CREATE TABLE + every migration step inside one sqlite transaction. python\'s sqlite3 module doesn\'t autocommit DDL by default, so if any later migration step throws on a user\'s specific DB shape (e.g. an old volume from a prior soulsync version with quirky schema state), the WHOLE batch rolls back — including the hifi_instances CREATE that ran successfully. user\'s next boot retries init, hits the same migration failure, rolls back again. table never lands. fix: defensive lazy-create. every hifi_instances CRUD method now runs `CREATE TABLE IF NOT EXISTS hifi_instances (...)` immediately before its operation. idempotent — costs one PRAGMA-level no-op when the table is already present, fully recovers from a broken init. read methods (`get_hifi_instances`, `get_all_hifi_instances`) now return empty instead of raising when init failed. write methods (`add`, `remove`, `toggle`, `reorder`, `seed`) work end-to-end. doesn\'t paper over the underlying init issue (still worth tracking down which migration breaks for which users) but makes hifi instance management self-healing. 7 new tests pin the lazy-create behavior — every method works against a DB that\'s missing the table.', page: 'settings' }, + { title: 'Plex: "All Libraries (Combined)" Mode', desc: 'github issue #505 (popebruhlxix): users with multiple plex music libraries (e.g. one per plex home user) only saw one library inside soulsync because the connection settings forced you to pick a single library section. now there\'s a new "all libraries (combined)" option in settings → connections → plex → music library dropdown. picking it flips the plex client into a server-wide read mode where every read method (`get_all_artists` / `get_all_album_ids` / `search_tracks` / `get_library_stats` / etc) dispatches through `server.library.search(libtype=...)` instead of querying a single section. one api call, plex handles the aggregation. cross-section dedup applied at the listing layer — same-name artists across sections collapse to a canonical entry (the one with more tracks), so plex home families with overlapping music tastes don\'t see "drake" twice. removal-detection id enumeration stays raw on purpose — deduping there would falsely prune tracks linked to non-canonical ratingKeys. write methods (genre / poster / metadata updates) are unaffected and operate on plex objects via ratingKey directly — write-back targets one section\'s copy of an artist if it exists in multiple, document and revisit if it matters. trigger_library_scan + is_library_scanning fan out across every music section in the new mode. backward compatible — existing users with a real library name saved see no behavior change. the "all libraries" option only appears in the dropdown when more than one music library exists on the server. 29 new tests pin both modes (single-section preserved, all-libraries dispatches through server-wide search, dedup keeps canonical, id enumeration stays raw).', page: 'settings' }, + { title: 'Fix: Download Discography Showed Wrong Artist\'s Albums', desc: 'clicked download discography on 50 cent → modal showed young hot rod\'s albums. clicked weird al → modal showed the beatles. cause: the endpoint received whichever single artist id the frontend happened to pick (spotify or itunes or deezer or library db id) and dispatched it as-is to whichever source it queried. when the picked id didn\'t match the queried source\'s id format, lookup either returned wrong-artist results (numeric collisions — db id 194687 was a real deezer artist for someone else) or fell back to a fuzzy name search that picked a wrong artist. the per-source id dispatch mechanism (`MetadataLookupOptions.artist_source_ids`) already existed and the watchlist scanner already used it; the on-demand discography endpoint just wasn\'t wired to it. fixed: when the url artist_id matches a library row by ANY stored id (db id, spotify_artist_id, itunes_artist_id, deezer_id, musicbrainz_id), backend pulls every stored provider id and dispatches the right id to each source. each source gets its OWN stored id regardless of what the url carries. when the url id is a non-library source-native id and the row lookup misses entirely, behavior is identical to before (single-id fallback). also fixed two log-namespace bugs: enhance quality and multi-source search were writing through `getLogger(__name__)` which resolves to `core.artists.quality` / `core.metadata.multi_source_search` — neither under the soulsync handler — so every diagnostic line was silently dropped. switched both to `get_logger()` from utils.logging_config so they actually land in app.log.', page: 'library' }, + { title: 'Enhance Quality: Direct ID Lookup Like Download Discography', desc: 'two related fixes. (1) discord report: enhance quality on an artist with neither spotify nor deezer connected added tracks as "unknown artist - unknown album - unknown track". enhance was running a single-source itunes fallback chain that returned junk matches with empty fields, while track redownload had been doing parallel multi-source search the whole time. extracted that search into `core/metadata/multi_source_search.py` and pointed both flows at it. (2) followup: enhance was still using fuzzy text search even when the library track had a stored source ID (spotify_track_id / deezer_id / itunes_track_id / soul_id) on the row, which meant tracks with messy tags ("Title (Live)", featured artists in the artist field, etc.) failed to match even though a perfect ID was sitting right there. download discography never had this problem because it resolves albums by stable ID, not by name. enhance now does the same: for every source you have configured, if the track has a stored ID for that source, it calls `get_track_details(id)` directly — no fuzzy matching. preferred source (your configured primary) is tried first so a deezer-primary user gets deezer payloads on the wishlist entry. text search is only the fallback now (kicks in for tracks with no stored IDs). also fixed the modal toast that lied "matching tracks to spotify..." regardless of which sources were actually being queried.', page: 'library' }, + { title: 'Internal: Media Server Engine Cin/JohnBaumb Pass', desc: 'internal — applied the same architectural cleanups the download engine PR went through to the media server engine PR before review. (1) every server client (Plex / Jellyfin / Navidrome / SoulSync) now explicitly inherits `MediaServerClient` instead of relying on structural typing — drift in any class fails at the conformance test boundary. (2) generic accessors on the engine: `configured_clients()` (replaces per-server `if X and X.is_connected(): clients[name] = X` chains in web_server.py) and `reload_config(name=None)` (generic dispatch instead of per-client reload calls). (3) singleton factory: `get_media_server_engine()` / `set_media_server_engine()` matching the metadata + download engine shape. web_server.py boots via `set_media_server_engine(...)` so factory + global handle share state. (4) ~70 direct `plex_client.X` / `jellyfin_client.X` / `navidrome_client.X` / `soulsync_library_client.X` attribute reaches in web_server.py migrated to `media_server_engine.client(\'\').X`. ~60 standalone refs (truthy checks, media_client assignments, source-name tuples) also routed through the engine. (5) the per-server `plex_client` / `jellyfin_client` / `navidrome_client` / `soulsync_library_client` globals in web_server.py are gone entirely — engine owns the client instances now, every caller reaches via `media_server_engine.client(\'\')`. four multi-client consumers (`PlaylistSyncService`, `ListeningStatsWorker`, `WebScanManager`, discovery `SyncDeps`) refactored to take the engine instead of separate per-server kwargs. (6) `TrackInfo` and `PlaylistInfo` lifted out of `core/plex_client.py` / `jellyfin_client.py` / `navidrome_client.py` (each was defining a near-identical copy) into the neutral `core/media_server/types.py` module — same lift Cin caught on the download `TrackResult`/`AlbumResult`/`DownloadStatus` situation. consumers (matching engine, sync service) get one import. zero behavior change.' }, + { title: 'Internal: Media Server Engine Foundation', desc: 'internal — companion to the download engine refactor. introduces a media-server engine + plugin contract on top of the four server clients (plex / jellyfin / navidrome / soulsync standalone). pre-refactor web_server.py held four separate per-server client globals that every dispatch site reached individually. new `core/media_server/` package provides `MediaServerEngine` that owns the per-server clients + a small set of generic accessors (`client(name)`, `active_client()`, `configured_clients()`, `reload_config(name)`) so call sites use one canonical lookup pattern. plugin contract requires the four methods every server actually implements (is_connected, ensure_connection, get_all_artists, get_all_album_ids) — methods that exist on most-but-not-all servers (search_tracks, trigger_library_scan, get_library_stats, etc.) are listed in `KNOWN_PER_SERVER_METHODS` for discoverability and reached directly via `engine.client(name).` since there\'s no uniform safe-default that fits every method. honest scope: the four uniform-shape `is_connected` dispatches were lifted into `engine.is_connected()` (the one cross-server wrapper kept on the engine); the ~18 server-specific chains that do genuinely different per-server work (playlist track replace, metadata sync, scan strategies) stay explicit at the call site per the "lift what\'s truly shared" standard, but reach the per-server client through the engine. 42 tests pin: per-server observable behavior (4 server pinning files, 20 tests), engine surface + accessor contracts (15 tests), structural conformance + explicit-inheritance (9 tests). zero behavior change for users.' }, + { title: 'Drop SoundCloud Preview Snippets at Search Time', desc: 'soundcloud serves a ~30s preview clip for tracks that are gated behind go+ / login (very common for major-label uploads — official content basically doesn\'t exist on soundcloud, so what shows up is bootlegs, fan uploads, type beats, and these 30s previews). yt-dlp accepts the preview as the download payload, the post-download integrity check catches the duration mismatch and quarantines the file, but the user just sees "all candidates failed" with no explanation. previews also showed up in the candidate-review modal where clicking one bypassed validation and downloaded the same broken file. now `filter_soundcloud_previews` drops these candidates at every entry point — auto-search scoring, modal-cache fallback, AND the not-found raw-results path — so previews never reach the matcher OR the user. drops candidates < 35s or below half the expected duration, gated on expected being non-trivially long (>60s) so genuine short tracks still pass. also fixed a silent regression in the hybrid-fallback path where the per-source attribute removal left `getattr(orch, \'youtube\', None)` returning None for every source — fallback never fired. now resolves through the orchestrator\'s `client(name)` accessor.', page: 'downloads' }, + { title: 'Internal: Move Shared Download Dataclasses + Singleton Boot Path', desc: 'internal — two architectural cleanups on top of the download engine refactor. (1) `TrackResult`, `AlbumResult`, `DownloadStatus`, `SearchResult` lived in `core/soulseek_client.py` for historical reasons (they grew up there as the soulseek-only types and got exported when other download sources were added). every plugin imported these from the soulseek module just to satisfy the contract — coupling 8 clients to a sibling source for type imports only. moved them to `core/download_plugins/types.py` (the neutral plugin package) and updated all 14 import sites across deezer/hifi/lidarr/qobuz/soundcloud/tidal/youtube clients + the engine + matching engine + redownload + tests. clean break, no backward-compat re-export. (2) `web_server.py` now boots the orchestrator via `set_download_orchestrator(DownloadOrchestrator())` so the singleton factory + boot path share state — `get_download_orchestrator()` returns the same instance the global handle points at instead of lazily building a separate one. matches cin\'s `get_metadata_engine()` pattern.' }, + { title: 'Internal: Rename `soulseek_client` Global → `download_orchestrator`', desc: 'internal — followup cleanup. the global handle in web_server.py was named `soulseek_client` for historical reasons (the orchestrator was originally just the soulseek client and grew downstream sources around it), but the type has long been `DownloadOrchestrator` not `SoulseekClient`. renamed the global + every parameter/attribute that carried the legacy name across web_server.py, api/, core/downloads/*, core/search/*, core/streaming/*, services/sync_service.py, and the test fixtures (`MasterDeps.soulseek_client` → `download_orchestrator`, `init(soulseek_client_obj)` → `init(download_orchestrator_obj)`, etc). module path `core.soulseek_client` and class `SoulseekClient` (the actual soulseek-only client) are unchanged — only the orchestrator handle renamed. ~250 references touched, suite green.' }, + { title: 'Internal: Drop Backward-Compat Per-Source Attrs', desc: 'internal — followup to cin\'s download engine review. removed the `orchestrator.soulseek` / `.youtube` / `.tidal` / `.qobuz` / `.hifi` / `.deezer_dl` / `.lidarr` / `.soundcloud` attribute aliases that were preserved for backward compat. external callers (core/downloads/, core/search/, web_server.py) all migrated to the generic `orchestrator.client(\'\')` accessor — alias-aware (legacy `deezer_dl` resolves to canonical `deezer`), single source of truth via the registry. the orchestrator\'s own internal `self.soulseek` / `self.deezer_dl` reaches also routed through `client()` so the only place that knows about per-source identity is the registry. test fakes updated to expose `client(name)` instead of stuffing attributes; conformance test pinned to the new accessor contract. zero behavior change — just cleaner shape.' }, + { title: 'Internal: Download Engine Review Followup', desc: 'internal — three correctness fixes on top of the download engine refactor, all flagged in cin\'s pr review. (1) `engine.cancel_download(source_hint=\'deezer_dl\')` was silently routing deezer cancels to soulseek because the legacy alias never made it to the engine\'s plugin map — only the registry knew about it. fix: aliases now flow through `register_plugin` and `get_plugin` / `cancel_download` resolve them to the canonical name. (2) `_resolve_source_chain` filtered hybrid_order against canonical registry names only, so any user with `deezer_dl` in their config quietly dropped deezer from hybrid mode. fix: orchestrator normalizes through `registry.get_spec()` first. (3) the worker\'s terminal write was a read-then-write split — a cancel landing between the snapshot and the update could be overwritten back to errored / completed. fix: new atomic `update_record_unless_state` on the engine holds `state_lock` across the check + write; both `_mark_terminal` AND the success path use it now. also added generic `client(name)` / `configured_clients()` / `reload_instances(name?)` accessors on the orchestrator + a `get/set_download_orchestrator()` singleton matching cin\'s `get_metadata_engine()` shape, and migrated 30 external `soulseek_client.` reaches in web_server.py to `client("")`. 18 new tests pin every fix.' }, + { title: 'Internal: Typed Metadata Foundation', desc: 'internal — first step of a multi-pr migration to give the metadata pipeline a real contract. the codebase historically grew duck-typed extractors (`_extract_lookup_value(album_data, "id", "album_id", "collectionId", "release_id", default=...)`) at every consumer site because each provider returns its own response shape. ~150 of those across the codebase. new `core/metadata/types.py` defines canonical typed `Album` / `Track` / `Artist` dataclasses with strict required fields. per-source classmethod converters (from_spotify_dict, from_itunes_dict, from_deezer_dict, from_discogs_dict, from_musicbrainz_dict, from_hydrabase_dict) are the SINGLE place that knows each provider\'s wire shape. zero behavior changes in this pr — pure additive foundation. follow-up prs migrate consumers one at a time. full migration plan documented at docs/metadata-types-migration.md.', page: 'library' }, + { title: 'Internal: Migrate Album-Info Builders to Typed Path', desc: 'internal — steps 2+3 of the typed metadata migration in one pr. two album-info builders now route through `Album.from__dict()` when the caller passes a known source: `_build_album_info` (used by every album-tracks lookup) and the embedded album section of `_build_single_import_context_payload` (used by single-track import context resolution). legacy duck-typed extraction stays as the fallback when source is empty/unknown, raw input isn\'t a dict, or the typed converter raises — so a converter bug can\'t break album resolution or import context. caller-provided album_id / album_name / artist_name fallbacks apply on the typed path the same way they did on legacy. zero behavior change for existing callers since they don\'t pass a source yet — opt-in only. 22 new tests pin the typed path, the legacy fallback, and parametrized coverage across registered providers.' }, + { title: 'Internal: Migrate Discography + Quality Scanner to Typed Path', desc: 'internal — next round of the typed metadata migration. three more album-shape consumers now route through `Album.from__dict()` when the caller passes a known source: `_build_discography_release_dict` (artist discography release cards), `_build_artist_detail_release_card` (artist detail page release cards), and `_normalize_track_album` (quality scanner result normalization). legacy duck-typed extraction stays as the fallback when source is empty/unknown, raw input isn\'t a dict, or the typed converter raises — same safety contract as the prior migration steps. 20 new tests pin the typed path + legacy fallback + parametrized coverage across registered providers.' }, + { title: 'Fix: Maintenance Findings Badge Showed Inflated Count With Empty Findings Tab', desc: 'discord report (husoyo): duplicate detector and cover art filler badges showed "364 findings" / "31 findings" after a scan, but clicking into the findings tab showed nothing. cause: `_create_finding` silently dedup-skipped re-discovered issues (when an equivalent row already existed with status pending/resolved/dismissed) but the caller incremented `result.findings_created` regardless of whether a row was actually inserted. so on a re-scan that found the same problems as a prior scan, the badge snapshot recorded 364 even though zero NEW pending rows hit the db. fix: `_create_finding` now returns a bool (True on insert, False on dedup-skip / db error). all 16 repair jobs updated to only increment `findings_created` on True. new `findings_skipped_dedup` counter added to job results and surfaced in the scan log: "Done: 2791 scanned, 0 fixed, 0 findings (363 already existed), 0 errors" — so re-scans show a real count, and you can see at a glance how many findings were carried over from prior scans. also fixed a missing `job_id` kwarg in the album tag consistency job that was silently breaking finding creation for that scan. companion ux improvement: findings tab now auto-switches its status filter from "pending" to "all status" when 0 pending rows exist but resolved/dismissed/auto-fixed rows do — with a small notice so you can see what carried over instead of staring at an "all clear" empty state.', page: 'library' }, + { title: 'Internal: Download Source Plugin Contract', desc: 'internal — first step of a multi-step refactor on the multi-source download dispatcher. the orchestrator historically had 8 download sources (soulseek/youtube/tidal/qobuz/hifi/deezer/lidarr/soundcloud) hardcoded into 6+ different dispatch sites — `if username == "youtube" elif username == "tidal" elif ...` chains in `__init__`, search, download, get_all_downloads, cancel_download, etc. adding usenet (planned) would have meant 700+ lines of copy-paste across the same files. new `core/download_plugins/` package defines `DownloadSourcePlugin` (Protocol) — the canonical contract every source must satisfy: `is_configured`, `check_connection`, `search`, `download`, `get_all_downloads`, `get_download_status`, `cancel_download`, `clear_all_completed_downloads`. plus `DownloadPluginRegistry` — single source of truth for which sources exist, with name/alias resolution (legacy `deezer_dl` alias preserved). orchestrator now dispatches through the registry instead of hardcoded `[self.soulseek, self.youtube, ...]` lists. (note: this PR initially preserved `self.` attribute aliases for backward compat; followup commits in the same PR removed them — see the "Drop Backward-Compat Per-Source Attrs" entry above. external callers now reach individual clients via `orchestrator.client('')`.) zero behavior change for end users — pure additive foundation that lets future PRs extract shared logic (background thread workers, search query normalization, post-processing context) into the contract instead of copy-pasted across all 8 sources. 19 new tests pin every plugin class\'s structural conformance to the contract — drift in any source will fail at the test boundary instead of at runtime against a live download.' }, + { title: 'Internal: Download Engine — Background Worker, State, Fallback', desc: 'internal — followup to the download source plugin contract. lifts the duplicated thread-spawn boilerplate, per-source active_downloads dicts, and hybrid-fallback dispatch into a central `core/download_engine/` package. each streaming source (youtube, tidal, qobuz, hifi, deezer, soundcloud) used to hand-roll the same ~70 LOC of background thread management — semaphore-gated serialization, rate-limit sleep between downloads, state-dict updates for InProgress/Completed/Errored transitions, exception capture. ~490 LOC of copy-paste across 7 files. all of it gone now — `engine.worker.dispatch(source, target_id, impl_callable, ...)` owns thread spawning + semaphore + delay + state lifecycle. plugins provide only `_download_sync(download_id, target_id, display_name) → file_path`, the source-specific atomic download. per-source rate-limit policy declared via `RateLimitPolicy` (concurrency, delay) — engine reads at register time. cross-source state queries (`get_all_downloads`, `get_download_status`, `cancel_download`, `clear_all_completed_downloads`) read engine state directly instead of iterating per-source dicts. hybrid-mode search now goes through `engine.search_with_fallback(chain)` — same ordering / skip-unconfigured / swallow-per-source-exceptions semantics as before. every per-source migration commit gated by phase A pinning tests (54 tests across all 8 sources) so contract drift fails fast at the test boundary instead of at runtime against a live download. net: ~700 LOC removed across 6 client files, ~85 new engine + worker + rate-limit tests, suite green at every commit. zero behavior change for end users — same downloads, same lifecycle states, same hybrid mode. (per-source attribute aliases like `orchestrator.soulseek` were initially preserved here for backward compat; followup commits in the same PR cycle removed them — see the "Drop Backward-Compat Per-Source Attrs" entry above. soulseek-specific internals are now reached via `orchestrator.client(\'soulseek\')._make_request(...)`.) adding usenet now = one new client class + one registry entry, no orchestrator changes. follow-up: cin\'s metadata engine work may shape further refactors (e.g. extracting search retry / quality filter — left per-source for now since search code is genuinely 90% source-specific).' }, + { title: 'Discogs Collection in "Your Albums"', desc: 'discord request: pull your discogs collection into the your albums section on discover, similar to spotify liked albums. set your discogs personal access token on settings → connections (already there from prior work) and add discogs as one of the configured sources via the gear button on your albums. background fetcher pulls your full collection (all folders, all pages — capped at 5000 releases), normalizes artist names (strips discogs `(N)` disambiguation suffix), dedupes against any spotify/tidal/deezer-saved versions of the same album. clicking a discogs-only album opens with discogs context — full release detail (year, format, label, country, tracklist) from the /releases endpoint. clicking an album that exists in both your spotify saved AND discogs collection prefers spotify (download flow is more direct). discogs is physical-media-first so many releases won\'t have streaming equivalents — those still show in the grid but the modal flow may need to fall back to a name search to find a downloadable digital version.', page: 'discover' }, + { title: 'Drop Redundant "Your Spotify Library" Section on Discover', desc: 'discover page used to show two near-identical sections: "Your Albums" (cross-source aggregator across spotify/deezer/etc) AND "Your Spotify Library" (spotify-only). same UI, same grid, same filter / sort / download-missing controls — the spotify-only one was a strict subset of what your albums already covers. removed it. spotify saved albums still surface via the your albums section with spotify as one of its configured sources (gear button → configure sources). backend collection / storage is unchanged — the watchlist scanner still populates the spotify_library_albums cache for your albums to read.', page: 'discover' }, + { title: 'Library Disk Usage on Stats Page', desc: 'discord request (samuel [KC]): show how much disk space the library takes. new card on stats → system statistics shows total bytes + per-format breakdown (FLAC vs MP3 vs M4A bars). data comes from `tracks.file_size` populated during deep scan from whatever the media server already returns (plex MediaPart.size, jellyfin MediaSources[].Size, navidrome song.size, soulsync standalone os.path.getsize) — zero filesystem walk overhead. existing libraries see "Run a Deep Scan to populate" until the next deep scan fills in sizes; partial coverage shown as "X tracks measured (+Y pending)". migration is additive (NULL on legacy rows) so upgrading users have nothing to do.', page: 'stats' }, + { title: 'Fix: ReplayGain Wrote Same +52 dB Gain to Every Track', desc: 'noticed every downloaded track came out with `replaygain_track_gain: +52.00 dB` regardless of actual loudness. cause: parser used `re.search` which returned the FIRST `I:` (integrated loudness) reading from ffmpeg\'s ebur128 output. that\'s the per-window measurement at t=0.5s — almost always ~-70 LUFS because tracks start with silence/encoder padding. -18 (RG2 reference) - (-70) = +52 dB on every track. fix: parser now anchors to the `Summary:` block at the end of ffmpeg\'s output and reads the actual integrated loudness from there, not the silent-intro partial. defensive fallback uses the LAST per-window reading if Summary is missing (still better than the first). gains now reflect real per-track loudness.', page: 'downloads' }, + { title: 'Fix: Tracks Showed Completed When File Was Quarantined', desc: 'caught downloading kendrick mr morale: three tracks (rich interlude, savior interlude, savior) showed ✅ completed in the modal but were missing on disk. two layered bugs. (1) the post-process verification wrapper had a fallback that assumed success when no `_final_processed_path` was in context — but integrity-rejected files (which get quarantined instead of moved) leave that path unset, so the wrapper marked them complete. now wrapper explicitly checks `_integrity_failure_msg` and `_race_guard_failed` markers before the assume-success fallback. failed integrity = task marked failed, batch tracker notified with success=false. (2) acoustid skip-logic was too lenient — when fingerprint confidence was very high and either title OR artist matched a bit, it skipped verification with reason "likely same song in different language/script." that fired for english-vs-english by the same artist with the word "interlude" in both — same artist + 0.55 title sim = skip = wrong file accepted. tightened: skip now requires non-ASCII chars present (real language/script case) AND artist match, OR very high title similarity (≥0.80) AND artist match. english-vs-english with very different titles by same artist no longer skipped — verification correctly returns FAIL and the wrong file gets quarantined.', page: 'downloads' }, + { title: 'Stop Navidrome From Splitting Albums Over Inconsistent MBIDs', desc: 'discord report (samuel [KC]): tracks of the same album sometimes carry different MUSICBRAINZ_ALBUMID tags, which causes navidrome to split the album into multiple entries. two-part fix: (1) the MBID Mismatch Detector now does a second scan that groups tracks by db album, finds the consensus (most-common) album mbid, and flags dissenters — fix action rewrites the dissenter\'s tag to match. catches existing inconsistencies in your library. (2) root cause: per-track musicbrainz release lookups went through an in-memory cache that\'s capped at 4096 entries and dies on server restart, so big libraries / restarts could resolve different release ids for tracks of the same album. added a persistent sqlite-backed cache so a release mbid resolved ONCE for an album applies to every future track of that album for the install\'s lifetime. strictly additive: any failure in the persistent layer falls through to the live musicbrainz lookup exactly as before.', page: 'library' }, + { title: 'Lidarr: Right Track Lands on Disk + Profile Lookup Stops Failing', desc: 'lidarr is an album-grabber — when you ask for one track it grabs the whole album, then we pick the wanted track out. old code blindly took the first imported file as the result, so any track you asked for got mistagged as track 1 of the album. now matches the wanted title against lidarr\'s track list (with punctuation-tolerant fuzzy compare) and copies only that file. also fixed a hardcoded `metadataProfileId=1` that broke artist-add on installs where someone had renamed/recreated profiles, and a polling-loop bug where the inner break never escaped the outer poll loop so completion detection was delayed. settings tooltip updated to be honest: lidarr is best for full-album grabs and effectively a no-op for playlist sync (track searches return nothing useful, hybrid mode falls through to your other sources).', page: 'settings' }, + { title: 'SoundCloud as a Download Source', desc: 'discord request (toasti): some tracks (DJ mixes, sets, removed-from-spotify exclusives) only live on soundcloud. soundcloud now plugs into the existing download-source picker on settings → downloads — pick "SoundCloud Only" or include it in the hybrid order alongside soulseek / youtube / tidal / qobuz / hifi / deezer / lidarr. anonymous-only (no account needed); quality is whatever soundcloud serves anonymously, typically 128 kbps mp3 or aac depending on the upload. soundcloud doesn\'t expose lossless to anyone, so don\'t expect flac. follows the exact same wiring contract as every other download source — search dispatch, hybrid fallback, queue / cancel / clear, sidebar source label, provenance + library history all work plug-and-play.', page: 'settings' }, + { title: 'Fix Qobuz Connection Not Sticking After Login', desc: 'logging in via the qobuz connect button on settings showed "connected: (active)" but underneath an error said "qobuz not authenticated...", and the dashboard indicator stayed yellow. cause: two separate qobuz client instances run side by side (one for the auth flow, one for the enrichment worker) and login only updated the first one. now the worker\'s client gets synced from config the moment login / token / logout completes, so the dashboard indicator goes green and connection-test stops yelling.', page: 'settings' }, + { title: 'Fix Lossy Copy Not Deleting Original FLAC', desc: 'with lossy copy enabled and "delete original" turned on (you wanted an mp3-only library), every download still left both the flac and the converted mp3 sitting in the same folder. the setting was being read but never acted on during the conversion step. now the original gets removed right after a successful conversion, with a same-path safety check + graceful handling if the original is already gone or locked.', page: 'settings' }, + { title: 'Watchlist Stops Re-Downloading Tracks That Already Exist', desc: 'a track that was already on disk got re-downloaded by the watchlist on every scan because the library had stale album metadata for it (file tagged on the wrong album by an old import) and the album fuzzy comparison declared the track missing. now the watchlist also matches by stable external IDs (spotify / itunes / deezer / tidal / qobuz / musicbrainz / audiodb / hydrabase / isrc) before falling through to the fuzzy block — so any track whose tags or DB row carry a matching ID is recognized as already present regardless of album drift. provider-neutral, falls through to existing fuzzy logic for older imports without IDs.', page: 'watchlist' }, + { title: 'Persist Source IDs at Download Time + Backfill on Sync', desc: 'every download already collects spotify/itunes/deezer/tidal/qobuz/musicbrainz/audiodb/hydrabase/isrc IDs during post-processing, but for plex/jellyfin/navidrome users they got dropped on the floor — only enrichment workers eventually wrote them onto the tracks row, hours later. now those IDs persist to the track_downloads table immediately, the media-server sync code copies them onto the new tracks row the moment it gets created, and the watchlist scanner has a second-tier fallback to query provenance directly when the tracks row hasn\'t been synced yet. closes the enrichment-wait window — freshly downloaded files are recognizable on the very next watchlist scan instead of after enrichment catches up.', page: 'library' }, + { title: 'Fix Tidal Auth Error 1002 for Docker / Remote Access', desc: 'tidal returned error 1002 ("invalid redirect URI") on every authentication attempt for users accessing soulsync from a network IP. cause: when the redirect_uri config field was empty (which it usually was, because the UI just shows the default as a placeholder without saving it), the /auth/tidal route silently overrode the constructor default with a uri built from request.host — http://192.168.x.x:8889/tidal/callback. that didn\'t match what users had registered in their tidal developer portal (http://127.0.0.1:8889/tidal/callback per the docs and UI default), so tidal rejected the authorize request before users ever saw the consent screen. fix: drop the request-host fallback entirely. empty config now falls back to the constructor default that matches the documented portal registration. the existing post-auth swap-step instructions handle the docker/remote-access case as designed.', page: 'settings' }, + { title: 'Auto-Import: Live Per-Track Progress in History', desc: 'dropping an album into the staging folder used to leave the auto-import history blank for the entire processing window — sometimes 5+ minutes for a full album — because the database row only got written after every track was post-processed. now an in-progress row gets inserted up-front (status=processing) the moment processing starts, then updated to completed/failed when done. the status indicator + progress bar show "processing speak now — track 3/14: mine", and the history card itself gets a pulsing "Processing" badge, swaps its meta line to "track 3/14: mine", and highlights the currently-processing row in the expanded track list (with prior tracks dimmed as done). one row per album, not per track, so the history list stays clean.', page: 'import' }, + { title: 'Reject Broken Files from slskd Before Tagging', desc: 'slskd sometimes reports a download as complete when the file is actually broken — truncated transfer, corrupted FLAC frames, or the wrong file matched on a similar filename. those slipped through into the library and surfaced as "song plays for 5 seconds and stops" or "track shows the wrong duration in plex." now every download gets a fast integrity check after the file stabilizes but before tagging / library sync: file size sanity (catches 0-byte and stub transfers), mutagen parse (catches header damage and wrong-format-with-right-extension cases), and duration agreement against the metadata source\'s expected length within a 3-second tolerance (5s for tracks over 10 minutes). failed files get quarantined to `ss_quarantine/` with a JSON sidecar explaining the failure, and the download slot is freed so a retry from another candidate can run.', page: 'downloads' }, + { title: 'Auto-Import: Multi-Disc Albums + Featured-Artist Tag Handling', desc: 'two longstanding auto-import gaps that surfaced when a kendrick lamar deluxe rip got dropped into staging. (1) folders containing only `Disc 1/`, `Disc 2/` subfolders (no loose audio at the parent level) used to be invisible to the scanner — disc folders were only attached to a parent when the parent had its own loose tracks. now scanner treats a parent of disc-only subfolders as the album candidate. (2) tag identification grouped files by `(album, artist)` — but per-track artist often varies on albums with features ("kendrick lamar" vs "kendrick lamar, drake" vs "kendrick lamar, dr. dre"), which fragmented the consensus and rejected real albums. now groups by album first, picks the dominant artist within that album group; also prefers `albumartist` tag over per-track `artist` since the former is the album-level identity. as a defensive bonus, when the staging folder itself becomes the candidate (raw disc folders dropped at the root with no album wrapper), the folder-name fallback gets skipped — the name "Staging" was matching against random albums in the metadata source.', page: 'import' }, + { title: 'Album Completeness Auto-Fill Works on Docker / Shared Library Setups', desc: 'github issue #476 (gabistek): the "auto-fill" button on the album completeness findings page returned `Could not determine album folder from existing tracks` for every album on docker setups (and any setup where the media-server library lives somewhere other than the soulsync transfer/download folders). cause: the repair worker\'s path resolver only probed the transfer + download folders, ignoring the user-configured `library.music_paths` and the plex-reported library locations. that missing search space meant docker users — whose plex/jellyfin library is bind-mounted at `/music` while soulsync\'s transfer is at `/transfer` — got silent "file not found" results for every existing track. extracted the full resolver (with library + plex sources) into a shared `core/library/path_resolver.py` and wired it into all five repair-worker call paths plus the four jobs that had their own incomplete copy. side benefit: every other repair job (dead file cleaner, mbid mismatch detector, lossy converter, acoustid scanner, unknown artist fixer) also stops missing files in the media-server library mount.', page: 'library' }, + { title: 'Sidebar Library Button Shows Artist Breadcrumb', desc: 'when you open an artist detail page (from library, search, or the global search popover), the sidebar Library button now lights up and rewrites its label to "Library / Artist Name" — long names truncate with an ellipsis and the full name shows on hover. revertes to plain "Library" when you leave. purely visual, no functionality change.', page: 'library' }, + { title: 'Enrichment Bubble Routes Consolidated', desc: 'internal — every dashboard enrichment bubble (musicbrainz, spotify, itunes, deezer, discogs, audiodb, lastfm, genius, tidal, qobuz) used to hit its own per-service status / pause / resume route in web_server.py. unified them under a single registry-driven endpoint set: /api/enrichment//. spotify\'s rate-limit guard, lastfm/genius yield-override behavior, and tidal/qobuz extra status fields are encoded as data on the registry. 27 new tests cover the registry behavior.' }, + { title: 'Drop Old Per-Service Enrichment Routes', desc: 'internal — followup to the registry consolidation. now that the dashboard has cut over to /api/enrichment//, deleted the 30 hand-rolled per-service routes from web_server.py (musicbrainz/audiodb/discogs/deezer/spotify/itunes/lastfm/genius/tidal/qobuz status+pause+resume). ~510 lines gone from the monolith, no behavior change.' }, + ], '2.4.1': [ // --- May 1, 2026 — patch release --- { date: 'May 1, 2026 — 2.4.1 release' }, @@ -3749,6 +3786,96 @@ const WHATS_NEW = { // Section shape: { title, description, features: [bullet strings], // usage_note?: 'optional hint shown at the bottom' } const VERSION_MODAL_SECTIONS = [ + { + title: "Big Sync Sessions No Longer Wedge After 2-3 Hours", + description: "github issue #499 (bafoed): downloading a big initial sync from spotify playlists worked for 2-3 hours then silently stopped. 3 active tasks stuck in \"searching\" state, replaced every ~10 min, slskd ui showed no actual activity. only fix was restarting the container.", + features: [ + "• root cause: `aiohttp.ClientSession()` was constructed with no timeout — when slskd hung (overloaded / network blip / internal stall), the http call blocked forever and the worker thread blocked with it", + "• download executor only has 3 worker threads — once all 3 wedged on hung calls, no further downloads could start", + "• fix: bounded `aiohttp.ClientTimeout` (total 120s, connect 15s, sock_read 60s) on every slskd session — slskd metadata calls finish in seconds, so the timeout can\'t kill a real operation", + "• timeout fires → caught + logged + return None → caller treats as a normal failure (same code path as a 5xx response) → worker thread unblocks → executor stays healthy", + "• 3 new pinning tests on the timeout config + handler so future drift fails at the test boundary, not against a wedged executor in production", + ], + usage_note: "no settings to change — applies on next container restart", + }, + { + title: "Library Reorganize No Longer Mistakes Album Tracks for Singles", + description: "github issue #500 (bafoed): library reorganize repair job was moving album tracks like `01 - Christine F.flac` to single-template paths because of a fragile classification heuristic.", + features: [ + "• pre-rewrite the job had its own tag-reading + transfer-folder walk + template logic — used `is_album = (group_size > 1)` where group_size was the count of same-album tracks in the transfer folder being scanned", + "• when only one track of an album sat in transfer (rest already moved, or album tags varied slightly like \"Buds\" vs \"Buds (Bonus)\") → group size 1 → routed to single template → wrong destination", + "• fix: delegate to the per-album planner the artist-detail \"reorganize\" modal already uses — db-driven, knows the album has n tracks regardless of how many currently sit in transfer", + "• only iterates albums on the ACTIVE media server (matches what the artist-detail modal sees) — multi-server users (plex + jellyfin etc) won\'t accidentally have the job touch the inactive server\'s files", + "• apply mode dispatches to the existing reorganize queue → one code path for file move + post-processing + db update + sidecar", + "• albums missing a metadata source id get a single \"needs enrichment first\" finding instead of n per-track \"no source\" findings cluttering the ui", + "• dropped ~500 loc that was duplicated against the per-album logic — files in transfer with no db entry are now exclusively the orphan file detector\'s domain", + ], + usage_note: "no settings to change — applies on next library reorganize repair job run", + }, + { + title: "Enrich Now Honors Manual Album Matches", + description: "github issue #501 (tacobell444): manually matching an album then clicking enrich would overwrite your manual match with whatever the worker\'s name-search returned, or revert status to \"not found\". reorganize then read the wrong id and moved files to the wrong destination.", + features: [ + "• every per-source enrichment worker (spotify / itunes / deezer / tidal / qobuz) now reads its stored id column at the top of `_process_*_individual` — if present, fetch directly via that id and refresh metadata without touching the id", + "• fuzzy name search only runs as fallback for entities that have never been matched", + "• discogs / audiodb / musicbrainz already had inline stored-id fast paths and are left alone — same correct behavior, just inline", + "• lastfm / genius are name-based and don\'t store ids — no-op for those", + "• cin-shape lift: same fix in 5 workers gets exactly one shared helper at `core/enrichment/manual_match_honoring.py`, per-worker variability (column name / client method / response shape) plugs in via callbacks", + "• reorganize fixed indirectly — it always honored stored ids correctly, the bug was upstream in enrich corrupting the id", + ], + usage_note: "no settings to change — applies on next click of enrich on a manually-matched album or track", + }, + { + title: "HiFi Instance Add No Longer Errors With \"no such table\"", + description: "github issue #503 (hadshaw21): adding a hifi instance from downloader settings popped up `no such table: hifi_instances` even when the connection test passed.", + features: [ + "• root cause: the bulk db init runs every CREATE TABLE + every migration step inside one sqlite transaction — python\'s sqlite3 module doesn\'t autocommit DDL, so if any later migration throws on your DB shape, the WHOLE batch rolls back including the hifi_instances create that ran successfully", + "• fix: defensive lazy-create — every hifi_instances CRUD method now runs `CREATE TABLE IF NOT EXISTS` right before its operation", + "• idempotent — one no-op cost when the table is already there, fully self-heals when it isn\'t", + "• read methods return empty instead of raising; write methods work end-to-end", + "• doesn\'t paper over the underlying init issue (still worth tracking which migration breaks for which users) but makes hifi instance management work regardless", + ], + usage_note: "no settings to change — applies on next click of \"add\" in hifi instance settings", + }, + { + title: "Plex: Combine All Music Libraries Into One", + description: "github issue #505 (popebruhlxix): users with multiple plex music libraries (e.g. one per plex home user) only saw one library inside soulsync because settings forced you to pick a single library section.", + features: [ + "• new \"all libraries (combined)\" option in settings → connections → plex → music library dropdown — only shows up when your server has more than one music library", + "• picking it flips the plex client into server-wide read mode — every scan / search / library-stat call dispatches through `server.library.search()` instead of querying a single section", + "• one api call, plex handles the aggregation — no per-section iteration code on our side", + "• cross-section dedup at the listing layer — same-name artists across sections (e.g. plex home families that both have drake) collapse to one canonical entry in your library list, so no more visual duplicates", + "• removal detection stays on raw ratingKeys — deduping there would falsely prune tracks linked to non-canonical entries", + "• write methods (genre / poster / metadata updates) and playlists are unaffected — section-agnostic, operate on plex objects via ratingKey", + "• trigger_library_scan + is_library_scanning fan out across every music section in the new mode", + "• backward compatible — existing users with a single library saved see no behavior change", + ], + usage_note: "settings → connections → plex → music library → pick \"all libraries (combined)\"", + }, + { + title: "Download Discography No Longer Shows Wrong Artist", + description: "clicked download discog on 50 cent → modal showed young hot rod\'s albums. clicked weird al → modal showed beatles albums. real bug, not just data weirdness.", + features: [ + "• endpoint received whichever single artist id the frontend happened to pick and dispatched it as-is to whichever source it queried — when the picked id didn\'t match the queried source\'s id format, lookup either returned wrong-artist results (numeric id collisions) or fell back to a fuzzy name search that picked a different artist", + "• fixed: backend now looks up the library row by ANY stored id (db id, spotify, itunes, deezer, musicbrainz) and dispatches the correct stored id to each source — every source gets its OWN id regardless of what the frontend chose to send", + "• mechanism already existed (`MetadataLookupOptions.artist_source_ids`) and the watchlist scanner already used it — discog endpoint just wasn\'t wired to it", + "• also fixed two log-namespace bugs: enhance quality + multi-source search were writing to a logger with no handlers, so every diagnostic line was silently dropped — now lands in app.log where you can actually see them", + ], + usage_note: "no settings to change — applies on next click of download discography", + }, + { + title: "Enhance Quality Now Behaves Like Download Discography", + description: "discord report — clicking enhance quality on an artist with no spotify/deezer was adding tracks as \"unknown artist - unknown album - unknown track\". root issue ran deeper than that single edge case.", + features: [ + "• enhance used to fuzzy text search the configured primary source only — a single-source itunes fallback returning junk matches with empty fields, while track redownload had been doing parallel multi-source search the whole time", + "• extracted that search into a shared module; both enhance and redownload now hit every configured source in parallel and pick the cross-source best match", + "• bigger fix: enhance now uses stored source IDs (spotify_track_id / deezer_id / itunes_track_id / soul_id) the same way download discography uses album IDs — direct lookup against each source's API, no fuzzy text matching, no failures from messy tags like \"Title (Live)\" or featured artists in the artist field", + "• preferred source (your configured primary) tried first so a deezer-primary user gets deezer payloads on the wishlist entry", + "• text search is only the fallback now — kicks in only when no stored IDs exist for the track", + "• modal toast no longer lies \"matching tracks to spotify\" regardless of which sources are actually configured", + ], + usage_note: "no settings to change — applies on next click of enhance quality", + }, { title: "Watchlist No Longer Re-Downloads Compilations", description: "compilation / soundtrack tracks were getting redownloaded on every watchlist scan because the album-name fuzzy check failed on naming drift between spotify and your media server.", @@ -4171,7 +4298,7 @@ function _getLatestWhatsNewVersion() { const versions = Object.keys(WHATS_NEW) .filter(v => _compareVersions(v, buildVer) <= 0) .sort((a, b) => _compareVersions(b, a)); - return versions[0] || '2.4.1'; + return versions[0] || '2.4.2'; } function openWhatsNew() { @@ -4329,7 +4456,7 @@ function _updateHelperBadge() { const TROUBLESHOOT_RULES = [ { - selector: '#spotify-service-card .status-dot.disconnected, #spotify-service-card .status-dot.error', + selector: '#metadata-source-service-card .service-card-indicator.disconnected, #metadata-source-service-card .service-card-indicator.error', title: 'Metadata Source Disconnected', steps: [ 'Go to Settings → Connections and verify your API credentials', @@ -4340,7 +4467,7 @@ const TROUBLESHOOT_RULES = [ action: { label: 'Open Settings', fn: () => navigateToPage('settings') } }, { - selector: '#media-server-service-card .status-dot.disconnected, #media-server-service-card .status-dot.error', + selector: '#media-server-service-card .service-card-indicator.disconnected, #media-server-service-card .service-card-indicator.error', title: 'Media Server Disconnected', steps: [ 'Check that your media server (Plex/Jellyfin/Navidrome) is running', @@ -4351,7 +4478,7 @@ const TROUBLESHOOT_RULES = [ action: { label: 'Open Settings', fn: () => navigateToPage('settings') } }, { - selector: '#soulseek-service-card .status-dot.disconnected, #soulseek-service-card .status-dot.error', + selector: '#soulseek-service-card .service-card-indicator.disconnected, #soulseek-service-card .service-card-indicator.error', title: 'Download Source Disconnected', steps: [ 'Verify your Soulseek/download client is running and reachable', diff --git a/webui/static/init.js b/webui/static/init.js index 6b3b11eb..efd6baa1 100644 --- a/webui/static/init.js +++ b/webui/static/init.js @@ -2189,6 +2189,13 @@ function navigateToPage(pageId, options = {}) { const navButton = document.querySelector(`[data-page="${pageId}"]`); if (navButton) { navButton.classList.add('active'); + } else if (pageId === 'artist-detail') { + // Artist-detail is a "pseudo-page" reachable from Library, Search, + // or the global search popover — it has no [data-page] match. Treat + // it as a Library context so the sidebar still anchors the user + // somewhere instead of showing a blank active state. + const libraryBtn = document.querySelector('[data-page="library"]'); + if (libraryBtn) libraryBtn.classList.add('active'); } // Update pages @@ -2199,6 +2206,12 @@ function navigateToPage(pageId, options = {}) { currentPage = pageId; + // Refresh the Library button label so artist-detail shows a breadcrumb + // ("Library / Artist Name") and other pages show plain "Library". + if (typeof _updateSidebarLibraryBreadcrumb === 'function') { + _updateSidebarLibraryBreadcrumb(); + } + if (!options.skipPushState) { const urlPath = pageId === 'dashboard' ? '/' : '/' + pageId; if (window.location.pathname !== urlPath) { diff --git a/webui/static/library.js b/webui/static/library.js index 6c4e7a4c..783c16ad 100644 --- a/webui/static/library.js +++ b/webui/static/library.js @@ -707,6 +707,63 @@ let discographyFilterState = { ownership: 'all' // 'all', 'owned', 'missing' }; +// Maximum visible characters of an artist name in the sidebar Library +// breadcrumb. Names longer than this get truncated with an ellipsis so the +// nav button width stays consistent across the rest of the sidebar. +const _SIDEBAR_BREADCRUMB_ARTIST_MAXLEN = 14; + + +function _updateSidebarLibraryBreadcrumb() { + // Rewrite the Library nav button label between plain "Library" and a + // "Library / " breadcrumb depending on whether the user is on + // the artist-detail pseudo-page. Pure visual — touches no app state. + const btn = document.querySelector('[data-page="library"]'); + if (!btn) return; + const textEl = btn.querySelector('.nav-text'); + if (!textEl) return; + + const onArtistDetail = (typeof currentPage === 'string' && currentPage === 'artist-detail'); + const artistName = onArtistDetail ? (artistDetailPageState.currentArtistName || '') : ''; + + if (!onArtistDetail || !artistName) { + // Default state: plain "Library" label. Use textContent so we wipe + // any previously-injected breadcrumb spans cleanly. + textEl.textContent = 'Library'; + textEl.removeAttribute('data-breadcrumb'); + return; + } + + // Truncate long names so the button width stays consistent. + let display = artistName; + if (display.length > _SIDEBAR_BREADCRUMB_ARTIST_MAXLEN) { + display = display.slice(0, _SIDEBAR_BREADCRUMB_ARTIST_MAXLEN - 1).trimEnd() + '…'; + } + + // Render via inline spans so CSS can style the root / separator / context + // independently. Escape via textContent on individual spans. + textEl.setAttribute('data-breadcrumb', '1'); + textEl.textContent = ''; + const root = document.createElement('span'); + root.className = 'nav-text-root'; + root.textContent = 'Library'; + const sep = document.createElement('span'); + sep.className = 'nav-text-sep'; + sep.textContent = ' / '; + const ctx = document.createElement('span'); + ctx.className = 'nav-text-context'; + ctx.textContent = display; + ctx.title = artistName; // full name on hover + textEl.appendChild(root); + textEl.appendChild(sep); + textEl.appendChild(ctx); +} + +// Expose so init.js navigateToPage can call it without a circular import. +if (typeof window !== 'undefined') { + window._updateSidebarLibraryBreadcrumb = _updateSidebarLibraryBreadcrumb; +} + + // Friendly labels for the dynamic "← Back to X" button on the artist-detail page. // Page id (the value of currentPage) -> button label. const _ARTIST_DETAIL_BACK_LABELS = { @@ -895,6 +952,14 @@ function initializeArtistDetailPage() { async function loadArtistDetailData(artistId, artistName) { console.log(`🔄 Loading artist detail data for: ${artistName} (ID: ${artistId})`); + // Refresh the sidebar Library breadcrumb so it picks up the new artist + // name. Covers same-page navigation between artists (similar-artist + // chain) where navigateToPage doesn't fire because the page id stays + // 'artist-detail'. + if (typeof _updateSidebarLibraryBreadcrumb === 'function') { + _updateSidebarLibraryBreadcrumb(); + } + // Reset discography filters to defaults resetDiscographyFilters(); @@ -1376,32 +1441,126 @@ function updateArtistHeroSection(artist, discography) { } // Lazy-load top tracks sidebar - if (artist.lastfm_url || artist.lastfm_listeners) { - _loadArtistTopTracks(artist.name); - } + // Always try metadata-source top tracks (Spotify / Deezer); fall back to + // Last.fm playcount when the source can't deliver. Last.fm-only mode is + // display-only (no download action), matching the legacy behavior. + _loadArtistTopTracks(artist.name); } +// Source label shown in the sidebar title. +const _TOP_TRACKS_SOURCE_LABELS = { + spotify: 'Top Tracks (Spotify)', + deezer: 'Top Tracks (Deezer)', + lastfm: 'Popular on Last.fm', +}; + async function _loadArtistTopTracks(artistName) { const sidebar = document.getElementById('artist-hero-sidebar'); const container = document.getElementById('hero-top-tracks'); + const titleEl = document.getElementById('hero-sidebar-title'); + const downloadAllBtn = document.getElementById('hero-top-tracks-download-all'); if (!sidebar || !container) return; + sidebar.style.display = 'none'; + if (downloadAllBtn) downloadAllBtn.style.display = 'none'; + + const _fmtNum = (n) => { + if (!n || n <= 0) return '0'; + if (n >= 1000000) return (n / 1000000).toFixed(1).replace(/\.0$/, '') + 'M'; + if (n >= 1000) return (n / 1000).toFixed(1).replace(/\.0$/, '') + 'K'; + return n.toLocaleString(); + }; + + const _escAttr = (s) => (s || '').replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>'); + + // ── Pass 1: metadata-source top tracks (Spotify / Deezer) ── + // Returns full track objects (id, artists, album, etc) so each row gets + // a real download action via the existing wishlist-add flow. The + // backend gracefully reports `success=False` for sources that don't + // expose popularity ranking (iTunes / Discogs / MusicBrainz), so the + // sidebar can fall through to the Last.fm display-only mode below. + const artistId = artistDetailPageState.currentArtistId; + if (artistId) { + try { + const params = new URLSearchParams({ limit: '10' }); + const resp = await fetch(`/api/artist/${encodeURIComponent(artistId)}/top-tracks?${params}`); + if (resp.ok) { + const data = await resp.json(); + if (data && data.success && Array.isArray(data.tracks) && data.tracks.length > 0) { + if (titleEl) titleEl.textContent = _TOP_TRACKS_SOURCE_LABELS[data.source] || 'Top Tracks'; + + // Stash the resolved tracks on the container so the + // bulk-download button below can hand them to the + // existing wishlist modal without refetching. + container._topTracksPayload = { + source: data.source, + tracks: data.tracks, + artistName, + artistId, + }; + + container.innerHTML = data.tracks.map((t, i) => { + const trackName = t.name || ''; + const trackArtists = (t.artists && t.artists.length) + ? t.artists.map(a => (a && a.name) ? a.name : '').filter(Boolean).join(', ') + : artistName; + return ` +
+ ${i + 1} + + ${_escAttr(trackName)} + +
+ `; + }).join(''); + + container.onclick = (e) => { + const playBtn = e.target.closest('.hero-top-track-play'); + if (playBtn) { + e.stopPropagation(); + playStatsTrack(playBtn.dataset.track, playBtn.dataset.artist, ''); + return; + } + const dlBtn = e.target.closest('.hero-top-track-download'); + if (dlBtn) { + e.stopPropagation(); + const idx = parseInt(dlBtn.dataset.index, 10); + const payload = container._topTracksPayload; + if (payload && Number.isFinite(idx) && payload.tracks[idx]) { + _topTrackDownloadOne(payload.tracks[idx], payload.artistName); + } + } + }; + + // Wire the bulk "Download All" footer button + if (downloadAllBtn) { + downloadAllBtn.style.display = ''; + downloadAllBtn.onclick = (e) => { + e.stopPropagation(); + const payload = container._topTracksPayload; + if (payload) _topTrackDownloadAll(payload); + }; + } + + sidebar.style.display = ''; + return; + } + } + } catch (e) { + console.debug('Top tracks metadata-source fetch failed:', e); + } + } + + // ── Pass 2 (fallback): Last.fm playcount, display-only ── try { const resp = await fetch(`/api/artist/0/lastfm-top-tracks?name=${encodeURIComponent(artistName)}`); const data = await resp.json(); if (!data.success || !data.tracks || data.tracks.length === 0) { - sidebar.style.display = 'none'; return; } - const _fmtNum = (n) => { - if (!n || n <= 0) return '0'; - if (n >= 1000000) return (n / 1000000).toFixed(1).replace(/\.0$/, '') + 'M'; - if (n >= 1000) return (n / 1000).toFixed(1).replace(/\.0$/, '') + 'K'; - return n.toLocaleString(); - }; - - const _escAttr = (s) => (s || '').replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>'); + if (titleEl) titleEl.textContent = _TOP_TRACKS_SOURCE_LABELS.lastfm; + container._topTracksPayload = null; container.innerHTML = data.tracks.map((t, i) => `
${i + 1} @@ -1411,7 +1570,6 @@ async function _loadArtistTopTracks(artistName) {
`).join(''); - // Attach play handlers via delegation (avoids inline JS escaping issues) container.onclick = (e) => { const btn = e.target.closest('.hero-top-track-play'); if (btn) { @@ -1421,8 +1579,76 @@ async function _loadArtistTopTracks(artistName) { }; sidebar.style.display = ''; } catch (e) { - console.debug('Failed to load top tracks:', e); - sidebar.style.display = 'none'; + console.debug('Failed to load top tracks (Last.fm fallback):', e); + } +} + +// Per-row download — wishlist a single track using its full metadata. +async function _topTrackDownloadOne(track, artistName) { + try { + const trackArtists = (track.artists && track.artists.length) + ? track.artists + : [{ name: artistName }]; + const album = (track.album && typeof track.album === 'object') ? track.album : {}; + const resp = await fetch('/api/add-album-to-wishlist', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + track: { ...track, artists: trackArtists }, + artist: { id: artistDetailPageState.currentArtistId || '', name: artistName }, + album: album, + source_type: 'top_tracks', + source_context: { + artist_name: artistName, + album_name: album.name || '', + album_type: album.album_type || 'album', + }, + }), + }); + const data = await resp.json(); + if (data && data.success) { + showToast(`Added "${track.name}" to wishlist`, 'success'); + } else { + showToast(`Failed to wishlist "${track.name}": ${(data && data.error) || 'unknown'}`, 'error'); + } + } catch (e) { + console.error('top track wishlist add failed:', e); + showToast('Failed to add track to wishlist', 'error'); + } +} + +// Bulk download — open the standard download modal in PLAYLIST context, +// not album context. The virtualPlaylistId intentionally doesn't start +// with `artist_album_` / `enhanced_search_album_` / etc, so +// `startMissingTracksProcess` (downloads.js) sets is_album_download=false +// and the master worker skips injecting the wrapper as `_explicit_album_context`. +// Result: each track downloads using its own real album metadata, files +// land in the proper per-album folders on disk. +function _topTrackDownloadAll({ source, tracks, artistName, artistId }) { + const virtualPlaylistId = `top_tracks_${source}_${artistId || 'unknown'}`; + const playlistName = `${artistName} — Top Tracks`; + const wrapperAlbum = { + id: virtualPlaylistId, + name: playlistName, + album_type: 'compilation', + images: [], + total_tracks: tracks.length, + artists: [{ id: artistId || '', name: artistName }], + }; + const artistObj = { + id: artistId || '', + name: artistName, + source: source, + }; + if (typeof openDownloadMissingModalForArtistAlbum === 'function') { + // contextType='playlist' tells the modal to render the playlist + // hero (not the album hero); the playlist_id prefix above is what + // actually drives the per-track album-folder routing on download. + openDownloadMissingModalForArtistAlbum( + virtualPlaylistId, playlistName, tracks, wrapperAlbum, artistObj, true, 'playlist' + ); + } else { + showToast('Download modal not available', 'error'); } } diff --git a/webui/static/settings.js b/webui/static/settings.js index dfa092bb..274b2561 100644 --- a/webui/static/settings.js +++ b/webui/static/settings.js @@ -60,7 +60,7 @@ function syncMetadataSourceSelection(source) { function _isMetadataSourceSelectable(source) { if (source === 'spotify') { - return _lastServiceStatus?.spotify?.authenticated === true; + return _lastStatusPayload?.spotify?.authenticated === true; } if (source === 'discogs') { const token = document.getElementById('discogs-token'); @@ -173,7 +173,7 @@ function initializeSettings() { if (discogsTokenInput) { discogsTokenInput.addEventListener('input', () => { if (typeof syncPrimaryMetadataSourceAvailability === 'function') { - syncPrimaryMetadataSourceAvailability(_lastServiceStatus?.spotify || null); + syncPrimaryMetadataSourceAvailability(_lastStatusPayload?.spotify || null); } sanitizeMetadataSourceSelection({ quiet: true }); }); @@ -207,10 +207,10 @@ function initializeSettings() { // Test button event listeners removed - they use onclick attributes in HTML to avoid double firing if (typeof syncPrimaryMetadataSourceAvailability === 'function') { - syncPrimaryMetadataSourceAvailability(_lastServiceStatus?.spotify || null); + syncPrimaryMetadataSourceAvailability(_lastStatusPayload?.spotify || null); } - syncSpotifySettingsAuthState(_lastServiceStatus?.spotify || null); - syncMetadataSourceSelection(_lastServiceStatus?.spotify?.source); + syncSpotifySettingsAuthState(_lastStatusPayload?.spotify || null); + syncMetadataSourceSelection(_lastStatusPayload?.metadata_source?.source); sanitizeMetadataSourceSelection({ quiet: true }); if (metadataSourceSelect) { metadataSourceSelect.dataset.lastValidSource = metadataSourceSelect.value; @@ -408,7 +408,7 @@ async function applyServiceStatusGradients() { else header.appendChild(spinner); } }); - syncSpotifySettingsAuthState(_lastServiceStatus?.spotify || null); + syncSpotifySettingsAuthState(_lastStatusPayload?.spotify || null); } catch (e) { console.warn('[Settings Status] Failed to apply gradients:', e); } @@ -596,10 +596,11 @@ const HYBRID_SOURCES = [ { id: 'hifi', name: 'HiFi', icon: null, emoji: '🎶' }, { id: 'deezer_dl', name: 'Deezer', icon: 'https://www.svgrepo.com/show/519734/deezer.svg', emoji: '🎧' }, { id: 'lidarr', name: 'Lidarr', icon: null, emoji: '📦' }, + { id: 'soundcloud', name: 'SoundCloud', icon: 'https://www.svgrepo.com/show/452219/soundcloud.svg', emoji: '☁️' }, ]; let _hybridSourceOrder = ['soulseek', 'youtube']; -let _hybridSourceEnabled = { soulseek: true, youtube: true, tidal: false, qobuz: false, hifi: false, deezer_dl: false, lidarr: false }; +let _hybridSourceEnabled = { soulseek: true, youtube: true, tidal: false, qobuz: false, hifi: false, deezer_dl: false, lidarr: false, soundcloud: false }; let _hybridVisualOrder = null; // Full visual order including disabled sources function buildHybridSourceList() { @@ -978,6 +979,7 @@ async function loadSettingsData() { document.getElementById('embed-qobuz').checked = settings.qobuz?.embed_tags !== false; document.getElementById('embed-lastfm').checked = settings.lastfm?.embed_tags !== false; document.getElementById('embed-genius').checked = settings.genius?.embed_tags !== false; + document.getElementById('embed-hifi').checked = settings.hifi?.embed_tags !== false; // Load per-tag toggles from data-config attributes document.querySelectorAll('[data-config]').forEach(cb => { const path = cb.dataset.config.split('.'); @@ -986,7 +988,7 @@ async function loadSettingsData() { cb.checked = val !== false; }); // Apply service disabled state to child tags - ['spotify', 'itunes', 'musicbrainz', 'deezer', 'audiodb', 'tidal', 'qobuz', 'lastfm', 'genius'].forEach(svc => { + ['spotify', 'itunes', 'musicbrainz', 'deezer', 'audiodb', 'tidal', 'qobuz', 'lastfm', 'genius', 'hifi'].forEach(svc => { const master = document.getElementById('embed-' + svc); if (master) toggleServiceTags(master, svc); }); @@ -1474,6 +1476,7 @@ function updateDownloadSourceUI() { const hifiContainer = document.getElementById('hifi-download-settings-container'); const deezerDlContainer = document.getElementById('deezer-download-settings-container'); const lidarrContainer = document.getElementById('lidarr-download-settings-container'); + const soundcloudContainer = document.getElementById('soundcloud-download-settings-container'); hybridContainer.style.display = mode === 'hybrid' ? 'block' : 'none'; @@ -1495,6 +1498,7 @@ function updateDownloadSourceUI() { hifiContainer.style.display = activeSources.has('hifi') ? 'block' : 'none'; if (deezerDlContainer) deezerDlContainer.style.display = activeSources.has('deezer_dl') ? 'block' : 'none'; if (lidarrContainer) lidarrContainer.style.display = activeSources.has('lidarr') ? 'block' : 'none'; + if (soundcloudContainer) soundcloudContainer.style.display = activeSources.has('soundcloud') ? 'block' : 'none'; // Quality profile is Soulseek-only and downloads-tab-only const qualityProfileSection = document.getElementById('quality-profile-section'); @@ -1513,6 +1517,9 @@ function updateDownloadSourceUI() { if (activeSources.has('hifi')) { testHiFiConnection(); } + if (activeSources.has('soundcloud')) { + testSoundcloudConnection(); + } } function updateHybridSecondaryOptions() { @@ -1525,6 +1532,9 @@ function updateHybridSecondaryOptions() { { value: 'tidal', label: 'Tidal' }, { value: 'qobuz', label: 'Qobuz' }, { value: 'hifi', label: 'HiFi' }, + { value: 'deezer_dl', label: 'Deezer' }, + { value: 'lidarr', label: 'Lidarr' }, + { value: 'soundcloud', label: 'SoundCloud' }, ]; secondary.innerHTML = ''; @@ -2543,7 +2553,7 @@ async function saveSettings(quiet = false) { const discogsTokenInput = document.getElementById('discogs-token'); const discogsTokenPresent = !!discogsTokenInput?.value?.trim(); let metadataSource = metadataSourceSelect?.value || 'itunes'; - const spotifySessionActive = _lastServiceStatus?.spotify?.authenticated === true; + const spotifySessionActive = _lastStatusPayload?.spotify?.authenticated === true; if (metadataSource === 'spotify' && !spotifySessionActive) { metadataSource = 'deezer'; if (metadataSourceSelect) metadataSourceSelect.value = metadataSource; @@ -2653,6 +2663,10 @@ async function saveSettings(quiet = false) { quality: document.getElementById('hifi-download-quality').value || 'lossless', allow_fallback: document.getElementById('hifi-allow-fallback').checked, }, + hifi: { + embed_tags: document.getElementById('embed-hifi').checked, + tags: _collectServiceTags('hifi') + }, deezer_download: { quality: document.getElementById('deezer-download-quality').value || 'flac', arl: document.getElementById('deezer-download-arl').value || '', @@ -2662,6 +2676,11 @@ async function saveSettings(quiet = false) { url: document.getElementById('lidarr-url').value || '', api_key: document.getElementById('lidarr-api-key').value || '', }, + soundcloud_download: { + // No knobs yet — anonymous-only. Keeping the key present so + // future tier-2 OAuth wiring (Go+ session token) doesn't have + // to migrate existing configs. + }, qobuz: { quality: document.getElementById('qobuz-quality').value || 'lossless', embed_tags: document.getElementById('embed-qobuz').checked, @@ -3416,6 +3435,32 @@ async function testHiFiConnection() { } } +async function testSoundcloudConnection() { + const statusEl = document.getElementById('soundcloud-connection-status'); + if (!statusEl) return; + statusEl.textContent = 'Checking...'; + statusEl.style.color = '#aaa'; + try { + const resp = await fetch('/api/soundcloud/status'); + const data = await resp.json(); + if (data.available && data.reachable) { + statusEl.textContent = 'Connected (anonymous)'; + statusEl.style.color = '#4caf50'; + } else if (data.available) { + // Client up but the live probe failed — likely a SoundCloud + // outage or a transient yt-dlp parse error. Surface plainly. + statusEl.textContent = 'Reachable check failed — try again'; + statusEl.style.color = '#ff9800'; + } else { + statusEl.textContent = data.error || 'Unavailable'; + statusEl.style.color = '#f44336'; + } + } catch (e) { + statusEl.textContent = 'Connection error'; + statusEl.style.color = '#f44336'; + } +} + async function testLidarrConnection() { const statusEl = document.getElementById('lidarr-connection-status'); if (!statusEl) return; diff --git a/webui/static/shared-helpers.js b/webui/static/shared-helpers.js index 6fed9d88..7b5d25c5 100644 --- a/webui/static/shared-helpers.js +++ b/webui/static/shared-helpers.js @@ -252,12 +252,15 @@ function createSearchController({ state._initialized = true; // Resolve the user's configured primary source. + // /status is public; /api/settings is admin-only and returns 403 for + // non-admin profiles, which previously caused them to silently fall + // back to 'spotify' regardless of what admin had configured. try { - const resp = await fetch('/api/settings'); + const resp = await fetch('/status'); if (resp.ok) { - const settings = await resp.json(); - const cfg = settings.metadata && settings.metadata.fallback_source; - if (cfg && SOURCE_LABELS[cfg]) state.activeSource = cfg; + const status = await resp.json(); + const src = status && status.metadata_source; + if (src && SOURCE_LABELS[src]) state.activeSource = src; } } catch (_) { /* best-effort */ } if (!SOURCE_LABELS[state.activeSource]) state.activeSource = 'spotify'; @@ -3143,7 +3146,7 @@ async function fetchAndUpdateServiceStatus() { const data = await response.json(); // Cache for library status card - _lastServiceStatus = data; + _lastStatusPayload = data; if (typeof syncSpotifySettingsAuthState === 'function') { syncSpotifySettingsAuthState(data?.spotify || null); @@ -3156,12 +3159,12 @@ async function fetchAndUpdateServiceStatus() { } // Update service status indicators and text (dashboard) - updateServiceStatus('spotify', data.spotify); + updateServiceStatus('metadata-source', data.metadata_source, data.spotify); updateServiceStatus('media-server', data.media_server); updateServiceStatus('soulseek', data.soulseek); // Update sidebar service status indicators - updateSidebarServiceStatus('spotify', data.spotify); + updateSidebarServiceStatus('metadata-source', data.metadata_source, data.spotify); updateSidebarServiceStatus('media-server', data.media_server); updateSidebarServiceStatus('soulseek', data.soulseek); @@ -3232,14 +3235,16 @@ function getMetadataSourceLabel(source) { return 'Unmapped'; } -function getSpotifyStatusPresentation(statusData) { - const sourceLabel = getMetadataSourceLabel(statusData?.source); - const rateLimited = !!(statusData?.rate_limited && statusData?.rate_limit); - const cooldown = !!(statusData?.post_ban_cooldown > 0); - const sessionActive = statusData?.authenticated === true || (statusData?.authenticated === undefined && statusData?.source === 'spotify'); +function getMetadataSourcePresentation(metadataStatus, spotifyStatus) { + const source = metadataStatus?.source; + const sourceLabel = getMetadataSourceLabel(source); + const connected = metadataStatus?.connected === true; + const sessionActive = spotifyStatus?.authenticated === true || (source === 'spotify' && connected); + const rateLimited = !!(source === 'spotify' && spotifyStatus?.rate_limited && spotifyStatus?.rate_limit); + const cooldown = !!(source === 'spotify' && spotifyStatus?.post_ban_cooldown > 0); if (rateLimited) { - const remaining = statusData.rate_limit?.remaining_seconds || 0; + const remaining = spotifyStatus.rate_limit?.remaining_seconds || 0; return { statusClass: 'rate-limited', statusText: `Spotify paused \u2014 ${formatRateLimitDuration(remaining)}`, @@ -3250,7 +3255,7 @@ function getSpotifyStatusPresentation(statusData) { } if (cooldown) { - const remaining = statusData.post_ban_cooldown; + const remaining = spotifyStatus.post_ban_cooldown; return { statusClass: 'rate-limited', statusText: `Spotify recovering \u2014 ${formatRateLimitDuration(remaining)}`, @@ -3260,32 +3265,37 @@ function getSpotifyStatusPresentation(statusData) { }; } - if (statusData?.source && statusData.source !== 'spotify') { + if (source) { return { - statusClass: 'connected', - statusText: sourceLabel, - dotClass: 'connected', - dotTitle: sourceLabel, + statusClass: connected ? 'connected' : 'disconnected', + statusText: connected ? (source === 'spotify' ? `Connected (${metadataStatus?.response_time}ms)` : sourceLabel) : 'Disconnected', + dotClass: connected ? 'connected' : 'disconnected', + dotTitle: connected ? sourceLabel : 'Disconnected', sessionActive }; } return { - statusClass: 'connected', - statusText: `Connected (${statusData?.response_time}ms)`, - dotClass: 'connected', - dotTitle: '', + statusClass: 'disconnected', + statusText: 'Disconnected', + dotClass: 'disconnected', + dotTitle: 'Disconnected', sessionActive }; } -function updateServiceStatus(service, statusData) { +function updateServiceStatus(service, statusData, spotifyStatus = null) { + const serviceCard = document.getElementById(`${service}-service-card`); const indicator = document.getElementById(`${service}-status-indicator`); const statusText = document.getElementById(`${service}-status-text`); + if (serviceCard) { + serviceCard.dataset.statusReady = 'true'; + } + if (indicator && statusText) { - if (service === 'spotify') { - const presentation = getSpotifyStatusPresentation(statusData || {}); + if (service === 'metadata-source') { + const presentation = getMetadataSourcePresentation(statusData || {}, spotifyStatus || {}); indicator.className = `service-card-indicator ${presentation.statusClass}`; statusText.textContent = presentation.statusText; statusText.className = `service-card-status-text ${presentation.statusClass}`; @@ -3303,8 +3313,8 @@ function updateServiceStatus(service, statusData) { } // Update music source title based on active source - if (service === 'spotify' && statusData.source) { - const musicSourceTitleElement = document.getElementById('music-source-title'); + if (service === 'metadata-source' && statusData.source) { + const musicSourceTitleElement = document.getElementById('metadata-source-title'); if (musicSourceTitleElement) { const sourceName = getMetadataSourceLabel(statusData.source); musicSourceTitleElement.textContent = sourceName; @@ -3312,7 +3322,7 @@ function updateServiceStatus(service, statusData) { } // Keep the Spotify action buttons aligned with the actual auth session. - const spotifySessionActive = getSpotifyStatusPresentation(statusData || {}).sessionActive; + const spotifySessionActive = spotifyStatus?.authenticated === true; const authBtn = document.querySelector('button[onclick="authenticateSpotify()"]'); const disconnectBtn = document.getElementById('spotify-disconnect-btn'); if (authBtn) { @@ -3322,27 +3332,42 @@ function updateServiceStatus(service, statusData) { disconnectBtn.style.display = spotifySessionActive ? '' : 'none'; } - syncPrimaryMetadataSourceAvailability(statusData); + syncPrimaryMetadataSourceAvailability(spotifyStatus); + + const responseTimeElement = document.getElementById('metadata-source-response-time'); + if (responseTimeElement) { + const responseTime = statusData.response_time; + responseTimeElement.textContent = responseTime !== undefined && responseTime !== null + ? `Response: ${responseTime}ms` + : 'Response: --'; + } + + const testButton = document.querySelector('#metadata-source-service-card .service-card-button'); + if (testButton) { + const source = statusData.source || 'spotify'; + testButton.setAttribute('onclick', `testDashboardConnection('${source}')`); + } } // Update download source title on dashboard card if (service === 'soulseek' && statusData.source) { - const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Hybrid' }; + const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' }; const displayName = sourceNames[statusData.source] || 'Soulseek'; const titleEl = document.getElementById('download-source-title'); if (titleEl) titleEl.textContent = displayName; } } -function updateSidebarServiceStatus(service, statusData) { +function updateSidebarServiceStatus(service, statusData, spotifyStatus = null) { const indicator = document.getElementById(`${service}-indicator`); if (indicator) { + indicator.dataset.statusReady = 'true'; const dot = indicator.querySelector('.status-dot'); const nameElement = indicator.querySelector('.status-name'); if (dot) { - if (service === 'spotify') { - const presentation = getSpotifyStatusPresentation(statusData || {}); + if (service === 'metadata-source') { + const presentation = getMetadataSourcePresentation(statusData || {}, spotifyStatus || {}); dot.className = `status-dot ${presentation.dotClass}`; dot.title = presentation.dotTitle; } else { @@ -3361,8 +3386,8 @@ function updateSidebarServiceStatus(service, statusData) { } // Update music source name in sidebar based on active source - if (service === 'spotify' && statusData.source) { - const musicSourceNameElement = document.getElementById('music-source-name'); + if (service === 'metadata-source' && statusData.source) { + const musicSourceNameElement = document.getElementById('metadata-source-name'); if (musicSourceNameElement) { const sourceName = getMetadataSourceLabel(statusData.source); musicSourceNameElement.textContent = sourceName; @@ -3371,7 +3396,7 @@ function updateSidebarServiceStatus(service, statusData) { // Update download source name based on configured mode if (service === 'soulseek' && statusData.source) { - const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Hybrid' }; + const sourceNames = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', lidarr: 'Lidarr', soundcloud: 'SoundCloud', hybrid: 'Hybrid' }; const displayName = sourceNames[statusData.source] || 'Soulseek'; const sidebarName = document.getElementById('download-source-name'); if (sidebarName) sidebarName.textContent = displayName; diff --git a/webui/static/stats-automations.js b/webui/static/stats-automations.js index 23b6bd4d..c29eb4b1 100644 --- a/webui/static/stats-automations.js +++ b/webui/static/stats-automations.js @@ -200,6 +200,8 @@ async function loadStatsData() { // DB storage chart (separate fetch — not part of cached stats) _loadDbStorageChart(); + // Library disk usage (separate fetch — populated by deep scan) + _loadLibraryDiskUsage(); // Recent plays _renderRecentPlays(data.recent || []); @@ -387,6 +389,70 @@ async function _loadDbStorageChart() { } } +async function _loadLibraryDiskUsage() { + try { + const resp = await fetch('/api/stats/library-disk-usage'); + const data = await resp.json(); + if (!data.success) return; + _renderLibraryDiskUsage(data); + } catch (e) { + console.debug('Library disk usage load failed:', e); + } +} + +function _formatBytes(n) { + if (!n || n <= 0) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let i = 0; + let v = n; + while (v >= 1024 && i < units.length - 1) { v /= 1024; i++; } + return `${v.toFixed(v < 10 ? 2 : 1)} ${units[i]}`; +} + +function _renderLibraryDiskUsage(data) { + const totalEl = document.getElementById('stats-disk-total-value'); + const metaEl = document.getElementById('stats-disk-total-meta'); + const formatsEl = document.getElementById('stats-disk-formats'); + if (!totalEl || !metaEl || !formatsEl) return; + + if (!data.has_data || !data.total_bytes) { + totalEl.textContent = '—'; + metaEl.textContent = data.tracks_without_size > 0 + ? `Run a Deep Scan to populate (${data.tracks_without_size.toLocaleString()} tracks pending)` + : 'No tracks in library yet'; + formatsEl.innerHTML = ''; + return; + } + + totalEl.textContent = _formatBytes(data.total_bytes); + + const withSize = data.tracks_with_size || 0; + const withoutSize = data.tracks_without_size || 0; + const trackBits = `${withSize.toLocaleString()} tracks measured`; + const pendingBits = withoutSize > 0 + ? ` (+${withoutSize.toLocaleString()} pending next Deep Scan)` + : ''; + metaEl.textContent = trackBits + pendingBits; + + // Per-format bars sorted by size descending. Skip if no breakdown. + const formats = Object.entries(data.by_format || {}).sort((a, b) => b[1] - a[1]); + if (!formats.length) { formatsEl.innerHTML = ''; return; } + + const max = formats[0][1] || 1; + formatsEl.innerHTML = formats.map(([ext, bytes]) => { + const pct = Math.max(2, Math.round((bytes / max) * 100)); + return ` +
+ ${ext.toUpperCase()} +
+
+
+ ${_formatBytes(bytes)} +
+ `; + }).join(''); +} + function _renderDbStorageChart(tables, totalFileSize, method) { const canvas = document.getElementById('stats-db-storage-chart'); if (!canvas || typeof Chart === 'undefined') return; @@ -626,12 +692,13 @@ function importPageSwitchTab(tab) { // ── Auto-Import Tab ── let _autoImportPollInterval = null; let _autoImportFilter = 'all'; +let _autoImportLastStatus = null; function _autoImportStartPolling() { _autoImportStopPolling(); - _autoImportPollInterval = setInterval(() => { + _autoImportPollInterval = setInterval(async () => { if (importPageState.activeTab === 'auto') { - _autoImportLoadStatus(); + await _autoImportLoadStatus(); _autoImportLoadResults(); } }, 5000); @@ -672,6 +739,7 @@ async function _autoImportLoadStatus() { const res = await fetch('/api/auto-import/status'); const data = await res.json(); if (!data.success) return; + _autoImportLastStatus = data; const toggle = document.getElementById('auto-import-enabled'); const statusText = document.getElementById('auto-import-status-text'); @@ -684,9 +752,22 @@ async function _autoImportLoadStatus() { if (settingsRow) settingsRow.style.display = data.running ? '' : 'none'; if (scanNowBtn) scanNowBtn.style.display = data.running ? '' : 'none'; - // Live scan progress + // Live scan + per-track processing progress if (progressEl) { - if (data.current_status === 'scanning') { + if (data.current_status === 'processing') { + progressEl.style.display = ''; + if (progressText) { + const idx = data.current_track_index || 0; + const total = data.current_track_total || 0; + const trackName = data.current_track_name || ''; + const folder = data.current_folder || '...'; + if (total > 0) { + progressText.textContent = `Processing ${folder} — track ${idx}/${total}: ${trackName}`; + } else { + progressText.textContent = `Processing: ${folder}`; + } + } + } else if (data.current_status === 'scanning') { progressEl.style.display = ''; if (progressText) { const stats = data.stats || {}; @@ -699,6 +780,7 @@ async function _autoImportLoadStatus() { if (statusText) { if (data.paused) statusText.textContent = 'Paused'; + else if (data.current_status === 'processing') statusText.textContent = 'Processing...'; else if (data.current_status === 'scanning') statusText.textContent = 'Scanning...'; else if (data.running) { // Show last scan time @@ -713,7 +795,12 @@ async function _autoImportLoadStatus() { } statusText.textContent = watchText; } else statusText.textContent = 'Disabled'; - statusText.className = 'auto-import-status ' + (data.running ? (data.current_status === 'scanning' ? 'scanning' : 'active') : 'disabled'); + const _runningClass = data.current_status === 'scanning' + ? 'scanning' + : data.current_status === 'processing' + ? 'processing' + : 'active'; + statusText.className = 'auto-import-status ' + (data.running ? _runningClass : 'disabled'); } } catch (e) {} } @@ -784,17 +871,30 @@ async function _autoImportLoadResults() { 'needs_identification': 'Unidentified', 'failed': 'Failed', 'scanning': 'Scanning...', 'matched': 'Matched', 'rejected': 'Dismissed', 'approved': 'Approved', + 'processing': 'Processing', }; const statusIcons = { 'completed': '\u2713', 'pending_review': '\u26A0', 'needs_identification': '\u2717', 'failed': '\u2717', 'scanning': '\u231B', 'matched': '\u2713', 'rejected': '\u2715', 'approved': '\u2713', + 'processing': '\u29D7', }; const statusLabel = statusLabels[r.status] || r.status; const statusIcon = statusIcons[r.status] || ''; const statusClass = r.status === 'completed' ? 'completed' : r.status === 'pending_review' ? 'review' : - r.status === 'failed' || r.status === 'needs_identification' ? 'failed' : 'neutral'; + r.status === 'failed' || r.status === 'needs_identification' ? 'failed' : + r.status === 'processing' ? 'processing' : 'neutral'; + + // Live per-track progress for the row currently being processed. + // Match by folder_name since the worker only tracks one folder at a time. + const liveStatus = _autoImportLastStatus; + const isLiveProcessing = r.status === 'processing' + && liveStatus && liveStatus.current_status === 'processing' + && liveStatus.current_folder === r.folder_name; + const liveTrackIdx = isLiveProcessing ? (liveStatus.current_track_index || 0) : 0; + const liveTrackTotal = isLiveProcessing ? (liveStatus.current_track_total || 0) : 0; + const liveTrackName = isLiveProcessing ? (liveStatus.current_track_name || '') : ''; // Parse match data for track details let matchCount = 0, totalTracks = 0, trackDetails = []; @@ -813,7 +913,10 @@ async function _autoImportLoadResults() { } catch (e) {} } - const matchSummary = totalTracks > 0 ? `${matchCount}/${totalTracks} tracks` : `${r.total_files} files`; + let matchSummary = totalTracks > 0 ? `${matchCount}/${totalTracks} tracks` : `${r.total_files} files`; + if (isLiveProcessing && liveTrackTotal > 0) { + matchSummary = `track ${liveTrackIdx}/${liveTrackTotal}: ${liveTrackName}`; + } const methodLabels = { tags: 'Tags', folder_name: 'Folder Name', acoustid: 'AcoustID', filename: 'Filename' }; const methodLabel = methodLabels[r.identification_method] || r.identification_method || ''; @@ -845,9 +948,15 @@ async function _autoImportLoadResults() {
TrackMatched FileConf
- ${trackDetails.map(t => { + ${trackDetails.map((t, tIdx) => { const tConfClass = t.confidence >= 90 ? 'high' : t.confidence >= 70 ? 'medium' : 'low'; - return `
+ // 1-based liveTrackIdx — current row glows, prior rows dim as "done". + let rowState = ''; + if (isLiveProcessing && liveTrackIdx > 0) { + if (tIdx + 1 === liveTrackIdx) rowState = ' auto-import-track-row-active'; + else if (tIdx + 1 < liveTrackIdx) rowState = ' auto-import-track-row-done'; + } + return `
${escapeHtml(t.name)} ${escapeHtml(t.file)} ${t.confidence}% @@ -7560,7 +7669,7 @@ async function submitEnhanceQuality() { submitBtn.disabled = true; submitBtn.innerHTML = 'Processing...'; } - if (footerInfo) footerInfo.textContent = 'Matching tracks to Spotify and adding to wishlist...'; + if (footerInfo) footerInfo.textContent = 'Matching tracks across metadata sources and adding to wishlist...'; try { const resp = await fetch(`/api/library/artist/${_enhanceArtistId}/enhance`, { diff --git a/webui/static/style.css b/webui/static/style.css index a25df5ed..b49adeab 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -273,10 +273,6 @@ body { gap: 8px; position: sticky; top: 0; - /* Explicitly lift above the nav buttons. `.sidebar > *` sets z-index 1 on - every sidebar child, so without this the sticky header paints at the - same stacking level as the nav and loses to it on DOM order. */ - z-index: 2; overflow: hidden; flex-shrink: 0; @@ -491,6 +487,42 @@ body { font-weight: 600; } +/* Library breadcrumb — when on the artist-detail pseudo-page, the Library + button label rewrites to "Library / ". The accent color stays on + the "Library" root; the separator and artist name are dimmed so the + anchor word remains scannable at a glance. JS truncates long names and + sets the full name as a `title` attribute for hover. */ +.nav-text[data-breadcrumb] { + display: inline-flex; + align-items: baseline; + max-width: 100%; + min-width: 0; +} +.nav-text[data-breadcrumb] .nav-text-root { + color: inherit; + flex-shrink: 0; +} +.nav-text[data-breadcrumb] .nav-text-sep { + color: rgba(255, 255, 255, 0.4); + font-weight: 400; + flex-shrink: 0; +} +.nav-button.active .nav-text[data-breadcrumb] .nav-text-sep { + color: rgba(255, 255, 255, 0.45); +} +.nav-text[data-breadcrumb] .nav-text-context { + color: rgba(255, 255, 255, 0.75); + font-weight: 500; + font-style: italic; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-width: 0; +} +.nav-button.active .nav-text[data-breadcrumb] .nav-text-context { + color: rgba(255, 255, 255, 0.85); +} + /* Sidebar Spacer */ .sidebar-spacer { flex: 1; @@ -2784,6 +2816,19 @@ body.helper-mode-active #dashboard-activity-feed:hover { font-weight: 400; } +.status-indicator[data-status-ready="false"] .status-dot, +.status-indicator[data-status-ready="false"] .status-name, +.service-card[data-status-ready="false"] .service-card-title, +.service-card[data-status-ready="false"] .service-card-indicator, +.service-card[data-status-ready="false"] .service-card-status-text, +.service-card[data-status-ready="false"] .service-card-response-time { + visibility: hidden; +} + +#metadata-source-service-card[data-status-ready="false"] .service-card-button { + visibility: hidden; +} + /* ===================================== MAIN CONTENT AREA STYLING ===================================== */ @@ -24297,6 +24342,55 @@ div.artist-hero-badge { font-variant-numeric: tabular-nums; } +/* Per-row wishlist button — only shown on metadata-source rows + (Spotify / Deezer). Last.fm rows show the playcount instead. */ +.hero-top-track-download { + width: 22px; + height: 22px; + border-radius: 50%; + border: none; + background: rgba(255, 255, 255, 0.06); + color: rgba(255, 255, 255, 0.5); + font-size: 11px; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + opacity: 0; + transition: all 0.2s; +} +.hero-top-track:hover .hero-top-track-download { + opacity: 1; +} +.hero-top-track-download:hover { + background: rgba(var(--accent-rgb), 0.25); + color: rgb(var(--accent-rgb)); + transform: scale(1.1); +} + +/* Footer "Download All" button on the top-tracks sidebar — only shown + when metadata-source provided downloadable tracks. */ +.hero-top-tracks-download-all { + margin-top: 12px; + width: 100%; + padding: 8px 12px; + border: 1px solid rgba(var(--accent-rgb), 0.3); + border-radius: 8px; + background: rgba(var(--accent-rgb), 0.08); + color: rgb(var(--accent-rgb)); + font-size: 0.78em; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + cursor: pointer; + transition: all 0.2s; +} +.hero-top-tracks-download-all:hover { + background: rgba(var(--accent-rgb), 0.18); + border-color: rgba(var(--accent-rgb), 0.5); +} + /* Mobile responsiveness */ @media (max-width: 768px) { .artist-hero-section { @@ -39400,6 +39494,72 @@ div.artist-hero-badge { text-align: right; } +/* Library Disk Usage */ +.stats-disk-usage-wrap { + display: flex; + flex-direction: column; + gap: 14px; + margin-top: 8px; +} + +.stats-disk-total-row { + display: flex; + align-items: baseline; + gap: 16px; + flex-wrap: wrap; +} + +.stats-disk-total-value { + font-size: 28px; + font-weight: 700; + color: rgb(var(--accent-rgb)); +} + +.stats-disk-total-meta { + font-size: 12px; + color: rgba(255, 255, 255, 0.55); +} + +.stats-disk-formats { + display: flex; + flex-direction: column; + gap: 6px; +} + +.stats-disk-format-row { + display: grid; + grid-template-columns: 60px 1fr 80px; + align-items: center; + gap: 10px; + font-size: 12px; +} + +.stats-disk-format-name { + font-weight: 600; + color: rgba(255, 255, 255, 0.8); +} + +.stats-disk-format-bar { + height: 8px; + background: rgba(255, 255, 255, 0.05); + border-radius: 4px; + overflow: hidden; +} + +.stats-disk-format-fill { + height: 100%; + background: linear-gradient(90deg, + rgb(var(--accent-rgb)) 0%, + rgba(var(--accent-rgb), 0.6) 100%); + border-radius: 4px; +} + +.stats-disk-format-size { + text-align: right; + color: rgba(255, 255, 255, 0.55); + font-variant-numeric: tabular-nums; +} + /* Database Storage Chart */ .stats-db-storage-wrap { display: flex; @@ -50718,6 +50878,13 @@ tr.tag-diff-same { background: rgba(239, 68, 68, 0.10); color: #f87171; } +/* Historical "found in last scan" — present but already resolved / + dismissed. Muted so it doesn't read as urgent action like the + current-pending badge. */ +.repair-flow-badge.findings-historical { + background: rgba(148, 163, 184, 0.10); + color: #94a3b8; +} .repair-flow-arrow { color: rgba(255, 255, 255, 0.25); font-size: 12px; @@ -59674,6 +59841,7 @@ body.reduce-effects *::after { .auto-import-completed { border-left: 3px solid #4ade80; } .auto-import-review { border-left: 3px solid #fbbf24; } .auto-import-failed { border-left: 3px solid #f87171; } +.auto-import-processing { border-left: 3px solid #60a5fa; } .auto-import-card-art { width: 56px; height: 56px; @@ -59931,6 +60099,26 @@ body.reduce-effects *::after { .auto-import-badge-review { background: rgba(251,191,36,0.1); color: #fbbf24; } .auto-import-badge-failed { background: rgba(248,113,113,0.1); color: #f87171; } .auto-import-badge-neutral { background: rgba(255,255,255,0.05); color: rgba(255,255,255,0.4); } +.auto-import-badge-processing { + background: rgba(96,165,250,0.12); + color: #60a5fa; + animation: auto-import-badge-pulse 1.6s ease-in-out infinite; +} +@keyframes auto-import-badge-pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.55; } +} + +.auto-import-track-row-active { + background: rgba(96,165,250,0.08); + border-left: 2px solid #60a5fa; + padding-left: 6px; + margin-left: -8px; + border-radius: 3px; +} +.auto-import-track-row-active .auto-import-track-name { color: #60a5fa; font-weight: 600; } +.auto-import-track-row-done .auto-import-track-name, +.auto-import-track-row-done .auto-import-track-file { opacity: 0.4; } .auto-import-actions { display: flex; gap: 4px; margin-top: 4px; diff --git a/webui/static/wishlist-tools.js b/webui/static/wishlist-tools.js index 3d40bb63..15766b5d 100644 --- a/webui/static/wishlist-tools.js +++ b/webui/static/wishlist-tools.js @@ -6558,8 +6558,8 @@ function updateLibraryStatusCard(dbStats) { const isScanning = window._libraryStatusScanning || false; // Determine state - const serverConnected = _lastServiceStatus && _lastServiceStatus.media_server && _lastServiceStatus.media_server.connected; - const serverType = _lastServiceStatus && _lastServiceStatus.active_media_server; + const serverConnected = _lastStatusPayload && _lastStatusPayload.media_server && _lastStatusPayload.media_server.connected; + const serverType = _lastStatusPayload && _lastStatusPayload.active_media_server; const hasData = tracks > 0; const hasServer = !!serverType && serverType !== 'none'; @@ -6668,7 +6668,7 @@ function updateLibraryStatusCard(dbStats) { } } -// _lastServiceStatus and _isSoulsyncStandalone are declared in core.js +// _lastStatusPayload and _isSoulsyncStandalone are declared in core.js const _origFetchServiceStatus = typeof fetchAndUpdateServiceStatus === 'function' ? fetchAndUpdateServiceStatus : null; function _capitalize(s) { return s ? s.charAt(0).toUpperCase() + s.slice(1) : ''; }