diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index fcb9638b..89f37e58 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,9 +9,9 @@ on: workflow_dispatch: inputs: version_tag: - description: 'Version tag (e.g. 2.5.2)' + description: 'Version tag (e.g. 2.6.0)' required: true - default: '2.5.2' + default: '2.6.0' jobs: build-and-push: diff --git a/.gitignore b/.gitignore index 8266b83b..8857f8d5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ database/music_library.db-shm database/music_library.db-wal database/music_library.db.backup_* database/api_call_history.json +storage/image_cache/ logs/*.log logs/*.log.* diff --git a/Dockerfile b/Dockerfile index a225b8b0..fdf5a21f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,8 +73,14 @@ COPY --chown=soulsync:soulsync --from=webui-builder /app/webui/static/dist /app/ # pre-baked directory would crash the container into a restart loop on # rootless Docker/Podman where in-container "root" can't write to /app. # Pre-baking the dir here makes the entrypoint mkdir a guaranteed no-op. -RUN mkdir -p /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/MusicVideos /app/scripts && \ - chown soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/MusicVideos /app/scripts +# NOTE: /app/Stream is the transient single-file streaming cache used by +# the basic-search "Play" flow (cleared per use, never persistent). It's +# created lazily by `core/streaming/prepare.py` via `os.makedirs`, which +# fails silently on rootless Docker where the soulsync UID can't write +# to /app — playback then errors out with no obvious cause. Pre-baking +# at build time (when the layer is owned by root) avoids that path. +RUN mkdir -p /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream /app/MusicVideos /app/scripts && \ + chown soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream /app/MusicVideos /app/scripts # Create defaults directory and copy template files # These will be used by entrypoint.sh to initialize empty volumes diff --git a/README.md b/README.md index f29f5c5f..a79eeb7e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ > **IMPORTANT**: Configure file sharing in slskd to avoid Soulseek bans. Set up shared folders at `http://localhost:5030/shares`. -**Community**: [Discord](https://discord.gg/wGvKqVQwmy) | [Reddit](https://old.reddit.com/r/ssync/) | **Website**: [ssync.net](https://www.ssync.net/) | **Support**: [GitHub Issues](https://github.com/Nezreka/SoulSync/issues) | **Donate**: [Ko-fi](https://ko-fi.com/boulderbadgedad) +**Community**: [Discord](https://discord.gg/wGvKqVQwmy) | **Website**: [ssync.net](https://www.ssync.net/) | **Support**: [GitHub Issues](https://github.com/Nezreka/SoulSync/issues) | **Donate**: [Ko-fi](https://ko-fi.com/boulderbadgedad) --- @@ -279,13 +279,34 @@ The template points at `boulderbadgedad/soulsync:latest` (stable) by default. To git clone https://github.com/Nezreka/SoulSync cd SoulSync python -m pip install -r requirements.txt + +# Build the React WebUI bundle used by the Python server. +# Docker does this automatically; Python installs must do it manually. +cd webui +npm ci +npm run build +cd .. + gunicorn -c gunicorn.conf.py wsgi:application # Open http://localhost:8008 ``` +When updating a Python/no-Docker install with `git pull`, rebuild the WebUI before restarting SoulSync: + +```bash +cd webui +npm ci +npm run build +cd .. +``` + +If `webui/static/dist/.vite/manifest.json` is missing or stale, React-owned routes and route handoffs may not load correctly. + ### Local Development -Use two terminals so the backend and Vite stay independent: +This is only for contributors working on the WebUI with hot reload. Normal Python/no-Docker installs should build once with `npm run build` as shown above, then run only Gunicorn. + +For active frontend development, use two terminals so the backend and Vite stay independent: 1. Backend ```bash diff --git a/api/downloads.py b/api/downloads.py index 69a9846d..29a6989d 100644 --- a/api/downloads.py +++ b/api/downloads.py @@ -125,6 +125,9 @@ def register_routes(bp): if not soulseek: return api_error("NOT_AVAILABLE", "Soulseek client not configured.", 503) + current_app.logger.info( + f"[CancelTrigger:api.public_cancel] download_id={download_id} username={username}" + ) ok = run_async(soulseek.cancel_download(download_id, username, remove=True)) if ok: return api_success({"message": "Download cancelled."}) diff --git a/api/serializers.py b/api/serializers.py index f7b0c1be..2102818d 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -368,6 +368,7 @@ def serialize_similar_artist(obj, fields: Optional[Set[str]] = None) -> dict: "source_artist_id": d.get("source_artist_id"), "similar_artist_spotify_id": d.get("similar_artist_spotify_id"), "similar_artist_itunes_id": d.get("similar_artist_itunes_id"), + "similar_artist_musicbrainz_id": d.get("similar_artist_musicbrainz_id"), "similar_artist_name": d.get("similar_artist_name"), "similarity_rank": d.get("similarity_rank"), "occurrence_count": d.get("occurrence_count"), diff --git a/config/settings.py b/config/settings.py index 2cfc3eeb..78b019fc 100644 --- a/config/settings.py +++ b/config/settings.py @@ -87,6 +87,10 @@ class ConfigManager: 'soulseek.api_key', 'deezer_download.arl', 'lidarr_download.api_key', + 'prowlarr.api_key', + 'torrent_client.password', + 'usenet_client.api_key', + 'usenet_client.password', # Enrichment services 'listenbrainz.token', 'acoustid.api_key', @@ -476,11 +480,19 @@ class ConfigManager: "search_min_delay_seconds": 0, }, "download_source": { - "mode": "soulseek", # Options: "soulseek", "youtube", "tidal", "qobuz", "hifi", "hybrid" + "mode": "soulseek", # Options: "soulseek", "youtube", "tidal", "qobuz", "hifi", "hybrid", "torrent", "usenet" "hybrid_primary": "soulseek", # Legacy: primary source for hybrid mode "hybrid_secondary": "youtube", # Legacy: fallback source for hybrid mode "hybrid_order": [], # Ordered list of sources for hybrid mode (overrides primary/secondary) "stream_source": "youtube", # Options: "youtube" (instant, default), "active" (use download source; falls back to youtube if soulseek) + # Album-bundle (torrent / usenet single-source) poll tuning. + # Downloader is polled every N seconds until the release + # lands; whole job aborts at the timeout. Defaults match + # the previous hard-coded constants. Users on slow private + # trackers / large box sets can extend the timeout without + # editing source. + "album_bundle_poll_interval_seconds": 2.0, + "album_bundle_timeout_seconds": 6 * 60 * 60, # 6 hours }, "tidal_download": { "quality": "lossless", # Options: "low", "high", "lossless", "hires" @@ -519,6 +531,37 @@ class ConfigManager: "quality_profile": "Any", "cleanup_after_import": True, }, + # Prowlarr — indexer aggregator. Feeds the torrent / usenet + # download plugins. Not a standalone source. + "prowlarr": { + "url": "", + "api_key": "", + # Comma-separated list of indexer IDs to limit searches to. + # Empty = search all enabled indexers. + "indexer_ids": "", + }, + # Torrent client — receives .torrent / magnet URIs from the + # torrent download plugin. ``type`` picks which adapter to + # instantiate (qbittorrent | transmission | deluge). + "torrent_client": { + "type": "qbittorrent", + "url": "", + "username": "", + "password": "", + "category": "soulsync", + "save_path": "", + }, + # Usenet client — receives .nzb URLs / payloads. ``type`` + # picks the adapter (sabnzbd | nzbget). SABnzbd uses an + # API key; NZBGet uses username + password. + "usenet_client": { + "type": "sabnzbd", + "url": "", + "api_key": "", + "username": "", + "password": "", + "category": "soulsync", + }, "soundcloud_download": { # Anonymous-only for now — SoundCloud Go+ OAuth tier could be # added later, with credentials living under a "session" subkey @@ -552,6 +595,13 @@ class ConfigManager: "path": os.environ.get('DATABASE_PATH', 'database/music_library.db'), "max_workers": 5 }, + "image_cache": { + "enabled": True, + "path": "storage/image_cache", + "ttl_seconds": 2592000, + "failed_ttl_seconds": 21600, + "max_download_mb": 15 + }, "metadata_enhancement": { "enabled": True, "embed_album_art": True, diff --git a/core/acoustid_verification.py b/core/acoustid_verification.py index bd2e917c..d1b87918 100644 --- a/core/acoustid_verification.py +++ b/core/acoustid_verification.py @@ -16,6 +16,7 @@ from enum import Enum from utils.logging_config import get_logger from core.acoustid_client import AcoustIDClient from core.matching_engine import MusicMatchingEngine +from core.matching.version_mismatch import is_acceptable_version_mismatch from core.musicbrainz_client import MusicBrainzClient logger = get_logger("acoustid.verification") @@ -485,12 +486,33 @@ class AcoustIDVerification: expected_version = _detect_title_version(expected_track_name) matched_version = _detect_title_version(matched_title) if expected_version != matched_version: - msg = ( - f"Version mismatch: expected '{expected_track_name}' ({expected_version}) " - f"but file is '{matched_title}' ({matched_version})" - ) - logger.warning(f"AcoustID verification FAILED (version mismatch) - {msg}") - return VerificationResult.FAIL, msg + # Issue #607 (AfonsoG6): MusicBrainz often stores live + # recordings with bare titles ("Clarity") while the + # release entry carries the venue annotation ("Clarity + # (Live at Blossom Music Center, ...)"). The fingerprint + # correctly identifies the LIVE recording; only the + # title text is bare. Helper accepts the one-sided bare + # case when fingerprint + bare-title + artist all agree. + # Two-sided version mismatches (live vs remix etc) stay + # strict — those are genuinely different recordings. + if is_acceptable_version_mismatch( + expected_version, matched_version, + fingerprint_score=best_score, + title_similarity=title_sim, + artist_similarity=artist_sim, + ): + logger.info( + f"AcoustID version annotation differs (expected={expected_version}, " + f"matched={matched_version}) but fingerprint+title+artist all match — " + f"accepting (likely MB metadata gap on a live/version-annotated recording)" + ) + else: + msg = ( + f"Version mismatch: expected '{expected_track_name}' ({expected_version}) " + f"but file is '{matched_title}' ({matched_version})" + ) + logger.warning(f"AcoustID verification FAILED (version mismatch) - {msg}") + return VerificationResult.FAIL, msg # Step 5: Decide pass/fail based on similarity if title_sim >= TITLE_MATCH_THRESHOLD and artist_sim >= ARTIST_MATCH_THRESHOLD: @@ -593,12 +615,19 @@ class AcoustIDVerification: # downloading Mr. Morale: three tracks (Rich Interlude, Savior # Interlude, Savior) all received the wrong R.O.T.C audio file # because of this leak. - top = recordings[0] - top_title = top.get('title', '?') or '' - top_artist = top.get('artist', '?') or '' + # Use the BEST matching recording's strings here (not + # `recordings[0]`) so the failure message reports the same + # candidate the title/artist similarity scores came from. + # Issue #607 (AfonsoG6) example 1: the prior code mixed + # `recordings[0]`'s strings (which can be empty) with + # `best_rec`'s scores, producing nonsense reasons like + # "file identified as '' by '' (artist=100%)" when a later + # recording in the list scored well on artist. + display_title = matched_title or '?' + display_artist = matched_artist or '?' has_non_ascii = ( any(ord(c) > 127 for c in (expected_track_name or '')) - or any(ord(c) > 127 for c in top_title) + or any(ord(c) > 127 for c in display_title) ) language_script_skip = ( best_score >= 0.95 @@ -618,18 +647,16 @@ class AcoustIDVerification: ) msg = ( f"Title/artist mismatch but fingerprint confidence very high ({best_score:.2f}): " - f"AcoustID='{top_title}' by '{top_artist}', " + f"AcoustID='{display_title}' by '{display_artist}', " f"expected '{expected_track_name}' by '{expected_artist_name}' — " f"{reason}" ) logger.info(f"AcoustID verification SKIPPED (high confidence) - {msg}") return VerificationResult.SKIP, msg - # Low fingerprint score + no metadata match — file is likely wrong - # `top`, `top_title`, `top_artist` already resolved above for the - # skip-eligibility check. + # Low fingerprint score + no metadata match — file is likely wrong. msg = ( - f"Audio mismatch: file identified as '{top_title}' by '{top_artist}', " + f"Audio mismatch: file identified as '{display_title}' by '{display_artist}', " f"expected '{expected_track_name}' by '{expected_artist_name}' " f"(title={title_sim:.0%}, artist={artist_sim:.0%})" ) diff --git a/core/amazon_client.py b/core/amazon_client.py new file mode 100644 index 00000000..2414051e --- /dev/null +++ b/core/amazon_client.py @@ -0,0 +1,786 @@ +"""Amazon Music metadata client backed by a T2Tunes proxy instance. + +T2Tunes exposes the Amazon Music catalog (search, album metadata, stream info) +through a simple REST API. This module wraps those endpoints and normalises the +responses into the same Track / Artist / Album dataclass shape used by +DeezerClient and iTunesClient. + +Endpoints used: + GET /api/status + GET /api/amazon-music/search + GET /api/amazon-music/metadata + GET /api/amazon-music/media-from-asin + +Config keys (all optional — fall back to public defaults): + amazon.base_url T2Tunes instance URL (default: https://t2tunes.site) + amazon.country ISO-3166 country code (default: US) + amazon.preferred_codec Preferred audio codec (default: flac) +""" + +from __future__ import annotations + +import re +import threading +import time +from dataclasses import dataclass +from typing import Any, Dict, Iterator, List, Optional +from urllib.parse import urljoin + +import requests + +from config.settings import config_manager +from core.api_call_tracker import api_call_tracker +from utils.logging_config import get_logger + +logger = get_logger("amazon_client") + +# Strips featuring credits like "Artist feat. X", "Artist ft. Y" so artist +# deduplication works on the primary artist name only. +_FEAT_RE = re.compile(r'\s+(?:feat(?:uring)?\.?|ft\.?)\s+.*', re.IGNORECASE) + +# Strips the Explicit marker — explicit is treated as the default version. +# Clean/Edited/Censored stay in the name so users can distinguish them. +_EDITION_RE = re.compile(r'\s*[\[\(]explicit[\]\)]', re.IGNORECASE) + + +def _primary_artist(name: str) -> str: + return _FEAT_RE.sub('', name).strip() + + +def _strip_edition(name: str) -> str: + return _EDITION_RE.sub('', name).strip() + + +def _unslugify(name: str) -> str: + """Convert a slug-form artist ID (e.g. 'kendrick_lamar') to a search name.""" + return name.replace('_', ' ') + + +DEFAULT_BASE_URL = "https://t2tunes.site" +DEFAULT_COUNTRY = "US" +DEFAULT_CODEC = "flac" +MIN_API_INTERVAL = 0.5 # seconds — T2Tunes has no published rate limit + +_last_api_call: float = 0.0 +_api_call_lock = threading.Lock() + +_META_CACHE_TTL = 300 # seconds +_meta_cache: Dict[str, tuple] = {} # asin -> (fetched_at, meta_dict) +_meta_cache_lock = threading.Lock() + + +class AmazonClientError(RuntimeError): + """Raised on unrecoverable T2Tunes API errors.""" + + +# --------------------------------------------------------------------------- +# Dataclasses — field layout matches DeezerClient / iTunesClient exactly +# --------------------------------------------------------------------------- + +@dataclass +class Track: + id: str # Amazon track ASIN + name: str + artists: List[str] + album: str + duration_ms: int + popularity: int + preview_url: Optional[str] = None + external_urls: Optional[Dict[str, str]] = None + image_url: Optional[str] = None + release_date: Optional[str] = None + track_number: Optional[int] = None + disc_number: Optional[int] = None + album_type: Optional[str] = None + total_tracks: Optional[int] = None + isrc: Optional[str] = None + + @classmethod + def from_search_hit(cls, doc: Dict[str, Any]) -> "Track": + return cls( + id=str(doc.get("asin") or ""), + name=str(doc.get("title") or ""), + artists=[str(doc.get("artistName") or "Unknown Artist")], + album=str(doc.get("albumName") or ""), + duration_ms=int(doc.get("duration") or 0) * 1000, + popularity=0, + isrc=str(doc.get("isrc") or "") or None, + ) + + @classmethod + def from_stream_info( + cls, + stream: "T2TunesStreamInfo", + album_meta: Optional[Dict[str, Any]] = None, + ) -> "Track": + album = album_meta or {} + return cls( + id=stream.asin, + name=stream.title, + artists=[stream.artist] if stream.artist else ["Unknown Artist"], + album=stream.album, + duration_ms=0, + popularity=0, + image_url=album.get("image"), + release_date=album.get("release_date"), + total_tracks=album.get("trackCount"), + isrc=stream.isrc or None, + ) + + +@dataclass +class Artist: + id: str # Slugified artist name — T2Tunes exposes no artist IDs + name: str + popularity: int + genres: List[str] + followers: int + image_url: Optional[str] = None + external_urls: Optional[Dict[str, str]] = None + + @classmethod + def from_name(cls, name: str) -> "Artist": + slug = name.lower().replace(" ", "_") + return cls(id=slug, name=name, popularity=0, genres=[], followers=0) + + +@dataclass +class Album: + id: str # Amazon album ASIN + name: str + artists: List[str] + release_date: str + total_tracks: int + album_type: str + image_url: Optional[str] = None + external_urls: Optional[Dict[str, str]] = None + explicit: Optional[bool] = None + + @classmethod + def from_search_hit(cls, doc: Dict[str, Any]) -> "Album": + return cls( + id=str(doc.get("albumAsin") or doc.get("asin") or ""), + name=str(doc.get("albumName") or doc.get("title") or ""), + artists=[str(doc.get("artistName") or "Unknown Artist")], + release_date="", + total_tracks=0, + album_type="album", + ) + + @classmethod + def from_metadata(cls, album_meta: Dict[str, Any], asin: str = "") -> "Album": + return cls( + id=str(album_meta.get("asin") or asin or ""), + name=str(album_meta.get("title") or ""), + artists=[str(album_meta.get("artistName") or "Unknown Artist")], + release_date=str(album_meta.get("release_date") or album_meta.get("releaseDate") or ""), + total_tracks=int(album_meta.get("trackCount") or 0), + album_type="album", + image_url=album_meta.get("image"), + explicit=album_meta.get("explicit"), + ) + + +# --------------------------------------------------------------------------- +# Internal dataclasses for raw T2Tunes response parsing +# --------------------------------------------------------------------------- + +@dataclass(frozen=True) +class T2TunesSearchItem: + asin: str + title: str + artist_name: str + item_type: str + album_name: str = "" + album_asin: str = "" + duration_seconds: int = 0 + isrc: str = "" + + @property + def is_album(self) -> bool: + return "album" in self.item_type.lower() + + @property + def is_track(self) -> bool: + return "track" in self.item_type.lower() + + +@dataclass(frozen=True) +class T2TunesStreamInfo: + asin: str + streamable: bool + codec: str + format: str + sample_rate: Optional[int] + stream_url: str + decryption_key: Optional[str] # hex-encoded AES key; None when stream is clear + title: str = "" + artist: str = "" + album: str = "" + isrc: str = "" + cover_url: str = "" + track_number: Optional[int] = None + disc_number: Optional[int] = None + genre: str = "" + label: str = "" + date: str = "" + + @property + def has_decryption_key(self) -> bool: + return bool(self.decryption_key) + + +# --------------------------------------------------------------------------- +# Rate-limit enforcement +# --------------------------------------------------------------------------- + +def _rate_limit() -> None: + global _last_api_call + with _api_call_lock: + elapsed = time.monotonic() - _last_api_call + if elapsed < MIN_API_INTERVAL: + time.sleep(MIN_API_INTERVAL - elapsed) + _last_api_call = time.monotonic() + api_call_tracker.record_call("amazon") + + +# --------------------------------------------------------------------------- +# Main client +# --------------------------------------------------------------------------- + +class AmazonClient: + """T2Tunes-backed Amazon Music metadata and stream-info client.""" + + def __init__( + self, + base_url: Optional[str] = None, + country: Optional[str] = None, + preferred_codec: Optional[str] = None, + timeout: int = 30, + session: Optional[Any] = None, + ) -> None: + self.base_url = (base_url or config_manager.get("amazon.base_url", DEFAULT_BASE_URL)).rstrip("/") + self.country = (country or config_manager.get("amazon.country", DEFAULT_COUNTRY)).upper() + self.preferred_codec = ( + preferred_codec or config_manager.get("amazon.preferred_codec", DEFAULT_CODEC) + ).lower() + self.timeout = timeout + self.session: Any = session or requests.Session() + if isinstance(self.session, requests.Session): + self.session.headers.update({ + "Accept": "application/json", + "User-Agent": "SoulSync/1.0", + "Referer": self.base_url, + }) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def reload_config(self) -> None: + self.base_url = config_manager.get("amazon.base_url", DEFAULT_BASE_URL).rstrip("/") + self.country = config_manager.get("amazon.country", DEFAULT_COUNTRY).upper() + self.preferred_codec = config_manager.get("amazon.preferred_codec", DEFAULT_CODEC).lower() + + def is_authenticated(self) -> bool: + """Return True when the T2Tunes instance reports Amazon Music as up.""" + try: + return str(self.status().get("amazonMusic", "")).lower() == "up" + except AmazonClientError: + return False + + # ------------------------------------------------------------------ + # Low-level API wrappers + # ------------------------------------------------------------------ + + def status(self) -> Dict[str, Any]: + return self._get_json("/api/status") + + def search_raw(self, query: str, *, types: str = "track,album") -> List[T2TunesSearchItem]: + data = self._get_json( + "/api/amazon-music/search", + params={"query": query, "types": types, "country": self.country}, + ) + return list(self._iter_search_items(data)) + + def album_metadata(self, asin: str) -> Dict[str, Any]: + return self._get_json( + "/api/amazon-music/metadata", + params={"asin": asin, "country": self.country}, + ) + + def media_from_asin(self, asin: str, codec: Optional[str] = None) -> List[T2TunesStreamInfo]: + effective_codec = (codec or self.preferred_codec).lower() + data = self._get_json( + "/api/amazon-music/media-from-asin", + params={"asin": asin, "country": self.country, "codec": effective_codec}, + ) + if isinstance(data, list): + return [self._parse_stream_info(item) for item in data if isinstance(item, dict)] + if isinstance(data, dict): + return [self._parse_stream_info(data)] + raise AmazonClientError(f"Unexpected media-from-asin response type: {type(data).__name__}") + + # ------------------------------------------------------------------ + # Metadata interface — mirrors DeezerClient / iTunesClient signatures + # ------------------------------------------------------------------ + + def search_tracks(self, query: str, limit: int = 20) -> List[Track]: + _rate_limit() + items = self.search_raw(query, types="track") + track_pairs: List[tuple] = [] # (Track, album_asin) + seen_album_asins: List[str] = [] + for item in items: + if not item.is_track: + continue + track = Track.from_search_hit({ + "asin": item.asin, + "title": _strip_edition(item.title), + "artistName": _primary_artist(item.artist_name), + "albumName": _strip_edition(item.album_name), + "albumAsin": item.album_asin, + "duration": item.duration_seconds, + "isrc": item.isrc, + }) + track_pairs.append((track, item.album_asin)) + if item.album_asin and item.album_asin not in seen_album_asins: + seen_album_asins.append(item.album_asin) + if len(track_pairs) >= limit: + break + album_metas = self._fetch_album_metas(seen_album_asins[:5]) + tracks: List[Track] = [] + for track, album_asin in track_pairs: + if album_asin and album_asin in album_metas: + meta = album_metas[album_asin] + track.image_url = meta.get("image") + track.release_date = str(meta.get("release_date") or meta.get("releaseDate") or "") + track.total_tracks = meta.get("trackCount") + tracks.append(track) + return tracks + + def search_artists(self, query: str, limit: int = 20) -> List[Artist]: + _rate_limit() + items = self.search_raw(query, types="track") + seen: Dict[str, Artist] = {} + artist_album_asin: Dict[str, str] = {} # artist name → first album ASIN seen + for item in items: + name = _primary_artist(item.artist_name) + if not name: + continue + if name not in seen: + seen[name] = Artist.from_name(name) + if name not in artist_album_asin and item.album_asin: + artist_album_asin[name] = item.album_asin + if len(seen) >= limit: + break + # T2Tunes has no artist images — use an album cover as stand-in. + unique_asins = list({v for v in artist_album_asin.values()})[:5] + album_metas = self._fetch_album_metas(unique_asins) + for name, artist in seen.items(): + asin = artist_album_asin.get(name) + if asin and asin in album_metas: + artist.image_url = album_metas[asin].get("image") + return list(seen.values()) + + def search_albums(self, query: str, limit: int = 20) -> List[Album]: + _rate_limit() + items = self.search_raw(query, types="track") + album_candidates: List[tuple] = [] # (Album, asin) + seen_keys: set = set() + for item in items: + album_asin = item.album_asin + raw_name = item.album_name + if not raw_name: + continue + display_name = _strip_edition(raw_name) + artist = _primary_artist(item.artist_name) + # Collapse Explicit/Clean variants: same normalised name + artist = same album + dedup_key = (display_name.lower(), artist.lower()) + if dedup_key in seen_keys: + continue + seen_keys.add(dedup_key) + album = Album.from_search_hit({ + "albumAsin": album_asin, + "albumName": display_name, + "artistName": artist, + }) + album_candidates.append((album, album_asin)) + if len(album_candidates) >= limit: + break + album_metas = self._fetch_album_metas([a for _, a in album_candidates[:10]]) + albums: List[Album] = [] + for album, asin in album_candidates: + if asin in album_metas: + meta = album_metas[asin] + album.image_url = meta.get("image") + album.release_date = str(meta.get("release_date") or meta.get("releaseDate") or "") + album.total_tracks = int(meta.get("trackCount") or 0) + albums.append(album) + return albums + + def get_track_details(self, asin: str) -> Optional[Dict[str, Any]]: + """Return a Spotify-compatible dict for a single track ASIN.""" + _rate_limit() + try: + streams = self.media_from_asin(asin) + except AmazonClientError: + return None + if not streams: + return None + s = streams[0] + + album_data: Dict[str, Any] = {} + try: + meta = self.album_metadata(asin) + albums = meta.get("albumList") + if isinstance(albums, list) and albums and isinstance(albums[0], dict): + album_data = albums[0] + except AmazonClientError: + pass + + return { + "id": s.asin, + "name": _strip_edition(s.title), + "artists": [{"name": _primary_artist(s.artist), "id": ""}], + "album": { + "id": album_data.get("asin", ""), + "name": _strip_edition(s.album), + "images": [{"url": album_data["image"]}] if album_data.get("image") else [], + "release_date": album_data.get("release_date") or album_data.get("releaseDate") or s.date or "", + "total_tracks": album_data.get("trackCount", 0), + }, + "duration_ms": 0, + "popularity": 0, + "external_urls": {"amazon": f"https://music.amazon.com/albums/{asin}"}, + "track_number": s.track_number, + "disc_number": s.disc_number, + "isrc": s.isrc, + "is_album_track": True, + "raw_data": { + "codec": s.codec, + "format": s.format, + "sample_rate": s.sample_rate, + "streamable": s.streamable, + "has_decryption_key": s.has_decryption_key, + }, + } + + def get_album(self, asin: str, include_tracks: bool = True) -> Optional[Dict[str, Any]]: + """Return a Spotify-compatible album dict.""" + _rate_limit() + try: + meta = self.album_metadata(asin) + except AmazonClientError: + return None + + albums = meta.get("albumList") + if not isinstance(albums, list) or not albums: + return None + album = albums[0] if isinstance(albums[0], dict) else {} + + result: Dict[str, Any] = { + "id": asin, + "name": _strip_edition(album.get("title", "")), + "artists": [{"name": _primary_artist(album.get("artistName", "")), "id": ""}], + "release_date": album.get("release_date") or album.get("releaseDate") or "", + "total_tracks": album.get("trackCount", 0), + "album_type": "album", + "images": [{"url": album["image"]}] if album.get("image") else [], + "external_urls": {"amazon": f"https://music.amazon.com/albums/{asin}"}, + "label": album.get("label", ""), + } + if include_tracks: + tracks_data = self.get_album_tracks(asin) or { + "items": [], "total": 0, "limit": 50, "next": None, + } + result["tracks"] = tracks_data + # Backfill release_date from stream tags when album metadata lacks it. + if not result["release_date"]: + items = tracks_data.get("items") or [] + for item in items: + rd = item.get("release_date") or "" + if rd and len(rd) >= 4: + result["release_date"] = rd + break + return result + + def get_album_tracks(self, asin: str) -> Optional[Dict[str, Any]]: + """Return album tracks in Spotify pagination format.""" + _rate_limit() + try: + streams = self.media_from_asin(asin) + except AmazonClientError: + return None + + # media_from_asin has no duration — enrich from search results which do. + duration_map: Dict[str, int] = {} # track asin → duration_ms + if streams: + album_name = _strip_edition(streams[0].album) + artist_name = _primary_artist(streams[0].artist) + try: + search_items = self.search_raw( + f"{album_name} {artist_name}", types="track" + ) + for item in search_items: + if item.album_asin == asin and item.duration_seconds: + duration_map[item.asin] = item.duration_seconds * 1000 + except Exception as exc: + logger.debug("Duration backfill failed for ASIN %s: %s", asin, exc) + + items = [ + { + "id": s.asin, + "name": _strip_edition(s.title), + "artists": [{"name": _primary_artist(s.artist), "id": ""}], + "duration_ms": duration_map.get(s.asin, 0), + "track_number": s.track_number if s.track_number is not None else idx + 1, + "disc_number": s.disc_number, + "release_date": s.date or "", + "isrc": s.isrc, + } + for idx, s in enumerate(streams) + ] + return {"items": items, "total": len(items), "limit": 50, "next": None} + + def get_artist(self, artist_name: str) -> Optional[Dict[str, Any]]: + """Return a Spotify-compatible artist dict inferred from search results.""" + _rate_limit() + search_name = _unslugify(artist_name) + try: + items = self.search_raw(search_name, types="track") + except AmazonClientError: + return None + name_lower = search_name.lower() + match = next( + (i for i in items if _primary_artist(i.artist_name).lower() == name_lower), + next((i for i in items if name_lower in _primary_artist(i.artist_name).lower()), None), + ) + if not match: + return None + return { + "id": match.artist_name.lower().replace(" ", "_"), + "name": match.artist_name, + "genres": [], + "popularity": 0, + "followers": {"total": 0}, + "images": [], + "external_urls": {}, + } + + def get_artist_albums( + self, + artist_name: str, + album_type: str = "album,single", + limit: int = 200, + ) -> List[Album]: + """Return albums for an artist inferred from search results.""" + _rate_limit() + search_name = _unslugify(artist_name) + try: + items = self.search_raw(f"{search_name} album", types="track") + except AmazonClientError: + return [] + album_candidates: List[tuple] = [] # (Album, asin) + seen_asins: set = set() + name_lower = search_name.lower() + for item in items: + if not item.is_album: + continue + if _primary_artist(item.artist_name).lower() != name_lower: + continue + album_asin = item.album_asin or item.asin + if album_asin in seen_asins: + continue + seen_asins.add(album_asin) + album = Album.from_search_hit({ + "albumAsin": album_asin, + "albumName": _strip_edition(item.album_name or item.title), + "artistName": _primary_artist(item.artist_name), + }) + album_candidates.append((album, album_asin)) + if len(album_candidates) >= limit: + break + + # Fetch metadata for art, release_date, track_count, and type inference. + # No explicit cap here — search_raw naturally bounds results to ~20 items, + # so album_candidates is already small. + asins_to_fetch = [a for _, a in album_candidates] + metas = self._fetch_album_metas(asins_to_fetch) + + albums: List[Album] = [] + for album, asin in album_candidates: + meta = metas.get(asin, {}) + if meta: + album.image_url = meta.get("image") + album.release_date = str(meta.get("release_date") or meta.get("releaseDate") or "") + total = int(meta.get("trackCount") or 0) + album.total_tracks = total + # T2Tunes doesn't expose release type — infer from track count. + # 1-track releases are singles; keep default "album" otherwise. + if total == 1: + album.album_type = "single" + albums.append(album) + return albums + + def get_track_features(self, track_id: str) -> Optional[Dict[str, Any]]: + """Not available from Amazon Music — returns None for compatibility.""" + return None + + def _get_artist_image_from_albums(self, artist_id: str) -> Optional[str]: + """Return an album cover as artist image stand-in (T2Tunes has no artist images).""" + search_name = _unslugify(artist_id) + try: + items = self.search_raw(search_name, types="track") + except AmazonClientError: + return None + name_lower = search_name.lower() + for item in items: + if _primary_artist(item.artist_name).lower() != name_lower: + continue + asin = item.album_asin or item.asin + if not asin: + continue + metas = self._fetch_album_metas([asin]) + if asin in metas and metas[asin].get("image"): + return metas[asin]["image"] + return None + + # ==================== Interface Aliases (match DeezerClient method names) ==================== + get_album_metadata = get_album + get_artist_info = get_artist + get_artist_albums_list = get_artist_albums + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _fetch_album_metas(self, asins: List[str]) -> Dict[str, Dict[str, Any]]: + """Parallel-fetch album metadata for up to N ASINs. Returns {asin: albumList[0]}.""" + if not asins: + return {} + metas: Dict[str, Dict[str, Any]] = {} + + def _fetch(asin: str) -> None: + now = time.monotonic() + with _meta_cache_lock: + entry = _meta_cache.get(asin) + if entry and (now - entry[0]) < _META_CACHE_TTL: + metas[asin] = entry[1] + return + _rate_limit() + try: + raw = self.album_metadata(asin) + lst = raw.get("albumList") + if isinstance(lst, list) and lst and isinstance(lst[0], dict): + meta = lst[0] + with _meta_cache_lock: + _meta_cache[asin] = (time.monotonic(), meta) + metas[asin] = meta + except Exception as exc: + logger.debug("Album metadata fetch failed for ASIN %s: %s", asin, exc) + + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=min(len(asins), 5)) as pool: + list(pool.map(_fetch, asins)) + return metas + + def _get_json(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any: + url = urljoin(f"{self.base_url}/", path.lstrip("/")) + last_exc: Optional[Exception] = None + for attempt in range(3): + if attempt: + time.sleep(1.0 * attempt) + try: + resp = self.session.get(url, params=params, timeout=self.timeout) + resp.raise_for_status() + except requests.HTTPError as exc: + body = exc.response.text[:500].replace("\n", " ") + # T2Tunes returns 400 for transient Amazon-side failures — retry those. + if exc.response.status_code == 400 and "Failed to search" in exc.response.text: + logger.debug("T2Tunes transient 400 on attempt %d, retrying: %s", attempt + 1, url) + last_exc = AmazonClientError( + f"HTTP {exc.response.status_code} for {url} — body: {body!r}" + ) + continue + raise AmazonClientError( + f"HTTP {exc.response.status_code} for {url} — body: {body!r}" + ) from exc + except requests.RequestException as exc: + raise AmazonClientError(f"Request failed for {url}: {exc}") from exc + try: + return resp.json() + except ValueError as exc: + preview = resp.text[:200].replace("\n", " ") + raise AmazonClientError(f"Response not JSON for {url}: {preview!r}") from exc + raise last_exc # type: ignore[misc] + + @staticmethod + def _iter_search_items(response: Any) -> Iterator[T2TunesSearchItem]: + if not isinstance(response, dict): + raise AmazonClientError( + f"Unexpected search response type: {type(response).__name__}" + ) + for result in response.get("results") or []: + if not isinstance(result, dict): + continue + for hit in result.get("hits") or []: + if not isinstance(hit, dict): + continue + doc = hit.get("document") + if not isinstance(doc, dict): + continue + asin = str(doc.get("asin") or "") + if not asin: + continue + yield T2TunesSearchItem( + asin=asin, + title=str(doc.get("title") or ""), + artist_name=str(doc.get("artistName") or ""), + item_type=str(doc.get("__type") or ""), + album_name=str(doc.get("albumName") or ""), + album_asin=str(doc.get("albumAsin") or ""), + duration_seconds=int(doc.get("duration") or 0), + isrc=str(doc.get("isrc") or ""), + ) + + @staticmethod + def _parse_stream_info(item: Dict[str, Any]) -> T2TunesStreamInfo: + stream_info = item.get("streamInfo") if isinstance(item.get("streamInfo"), dict) else {} + tags = item.get("tags") if isinstance(item.get("tags"), dict) else {} + # T2Tunes API has a typo: "stremeable" in some responses + streamable = item.get("streamable") + if streamable is None: + streamable = item.get("stremeable") + raw_key = item.get("decryptionKey") + decryption_key = str(raw_key) if raw_key else None + + def _int_tag(key: str) -> Optional[int]: + v = tags.get(key) + try: + return int(v) if v is not None else None + except (TypeError, ValueError): + return None + + return T2TunesStreamInfo( + asin=str(item.get("asin") or ""), + streamable=bool(streamable), + codec=str(stream_info.get("codec") or ""), + format=str(stream_info.get("format") or ""), + sample_rate=( + stream_info.get("sampleRate") + if isinstance(stream_info.get("sampleRate"), int) + else None + ), + stream_url=str(stream_info.get("streamUrl") or ""), + decryption_key=decryption_key, + title=str(tags.get("title") or ""), + artist=str(tags.get("artist") or ""), + album=str(tags.get("album") or ""), + isrc=str(tags.get("isrc") or ""), + cover_url=str(item.get("coverUrl") or ""), + track_number=_int_tag("trackNumber"), + disc_number=_int_tag("discNumber"), + genre=str(tags.get("genre") or ""), + label=str(tags.get("label") or ""), + date=str(tags.get("date") or ""), + ) diff --git a/core/amazon_download_client.py b/core/amazon_download_client.py new file mode 100644 index 00000000..18e4bad1 --- /dev/null +++ b/core/amazon_download_client.py @@ -0,0 +1,452 @@ +"""Amazon Music download source plugin backed by a T2Tunes proxy. + +NOT yet wired into the app registry — validated in isolation only. +See tests/tools/test_amazon_download_client.py. + +Filename encoding (the DownloadSourcePlugin dispatch contract): + "{asin}||{display_name}" + e.g. "B09XYZ1234||Kendrick Lamar - Not Like Us" + +Codec preference order: FLAC → Opus → EAC3. + +Download flow (from Tubifarry reference implementation): + 1. GET stream_url → encrypted bytes on disk + 2. FFmpeg -decryption_key to decrypt in place + 3. Embed metadata tags (handled by the app's standard post-processing) +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import requests as http_requests + +from config.settings import config_manager +from core.amazon_client import AmazonClient, AmazonClientError +from core.download_plugins.base import DownloadSourcePlugin +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult +from utils.logging_config import get_logger + +logger = get_logger("amazon_download_client") + +# Quality / codec helpers +CODEC_PREFERENCE = ["flac", "opus", "eac3"] + +_CODEC_EXTENSIONS: Dict[str, str] = { + "flac": "flac", + "ogg_vorbis": "ogg", + "opus": "opus", + "eac3": "eac3", + "mp4": "m4a", + "aac": "m4a", + "mp3": "mp3", +} + +MIN_AUDIO_BYTES = 512 * 1024 # 512 KB — anything smaller is a broken stream + + +def _codec_key(codec: str) -> str: + return codec.lower().replace("-", "_").replace(" ", "_") + + +def _file_extension(codec: str) -> str: + return _CODEC_EXTENSIONS.get(_codec_key(codec), "bin") + + +def _quality_label(codec: str, sample_rate: Optional[int] = None) -> str: + ck = _codec_key(codec) + if ck == "flac": + if sample_rate and sample_rate > 48000: + return "Hi-Res" + return "Lossless" + return "Lossy" + + +class AmazonDownloadClient(DownloadSourcePlugin): + """DownloadSourcePlugin — Amazon Music via T2Tunes proxy.""" + + def __init__(self, download_path: Optional[str] = None) -> None: + if download_path is None: + download_path = config_manager.get("soulseek.download_path", "./downloads") + self.download_path = Path(download_path) + self.download_path.mkdir(parents=True, exist_ok=True) + + self._quality = config_manager.get("amazon_download.quality", "flac") + self._allow_fallback = config_manager.get("amazon_download.allow_fallback", True) + + self._client = AmazonClient(preferred_codec=self._quality) + self.session = http_requests.Session() + self.session.headers.update({ + "User-Agent": "SoulSync/1.0", + "Accept": "*/*", + }) + + self._engine: Any = None + self.shutdown_check: Any = None + + # ------------------------------------------------------------------ + # DownloadSourcePlugin — lifecycle + # ------------------------------------------------------------------ + + def set_engine(self, engine) -> None: + """Engine callback — wires the central thread worker + state store.""" + self._engine = engine + + def set_shutdown_check(self, check_callable) -> None: + self.shutdown_check = check_callable + + def is_configured(self) -> bool: + # T2Tunes has a public default instance; no credentials required. + # Return True unconditionally so the source shows as available. + return True + + async def check_connection(self) -> bool: + try: + return self._client.is_authenticated() + except Exception: + return False + + # ------------------------------------------------------------------ + # DownloadSourcePlugin — search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback: Any = None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + try: + items = self._client.search_raw(query, types="track,album") + except AmazonClientError as exc: + logger.warning(f"Amazon search failed for {query!r}: {exc}") + return [], [] + + track_results: List[TrackResult] = [] + album_map: Dict[str, AlbumResult] = {} + album_order: List[str] = [] + preferred = self._client.preferred_codec + + for item in items: + quality = _quality_label(preferred) + if item.is_track: + track_results.append(TrackResult( + username="amazon", + filename=f"{item.asin}||{item.artist_name} - {item.title}", + size=0, + bitrate=None, + duration=item.duration_seconds * 1000 if item.duration_seconds else None, + quality=quality, + free_upload_slots=999, + upload_speed=999_999, + queue_length=0, + artist=item.artist_name, + title=item.title, + album=item.album_name, + _source_metadata={ + "asin": item.asin, + "album_asin": item.album_asin, + "isrc": item.isrc, + }, + )) + elif item.is_album: + album_asin = item.album_asin or item.asin + if album_asin not in album_map: + placeholder = TrackResult( + username="amazon", + filename=f"{item.asin}||{item.artist_name} - {item.title}", + size=0, + bitrate=None, + duration=None, + quality=quality, + free_upload_slots=999, + upload_speed=999_999, + queue_length=0, + artist=item.artist_name, + title=item.title, + album=item.album_name, + ) + album_map[album_asin] = AlbumResult( + username="amazon", + album_path=album_asin, + album_title=item.album_name or item.title, + artist=item.artist_name, + track_count=0, + total_size=0, + tracks=[placeholder], + dominant_quality=quality, + ) + album_order.append(album_asin) + + return track_results, [album_map[k] for k in album_order] + + # ------------------------------------------------------------------ + # DownloadSourcePlugin — download dispatch + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + if "||" not in filename: + logger.error(f"Invalid Amazon filename format (no '||'): {filename!r}") + return None + if self._engine is None: + raise RuntimeError( + "AmazonDownloadClient._engine is not set — " + "client not connected to download infrastructure" + ) + asin, display_name = filename.split("||", 1) + asin = asin.strip() + display_name = display_name.strip() + return self._engine.worker.dispatch( + source_name="amazon", + target_id=asin, + display_name=display_name, + original_filename=filename, + impl_callable=self._download_sync, + ) + + def _download_sync( + self, + download_id: str, + target_id: str, + display_name: str, + ) -> Optional[str]: + asin = str(target_id) + codecs = CODEC_PREFERENCE if self._allow_fallback else [self._quality] + for codec in codecs: + try: + streams = self._client.media_from_asin(asin, codec=codec) + except AmazonClientError as exc: + logger.warning(f"media_from_asin({asin!r}, {codec}) failed: {exc}") + continue + + stream = next( + (s for s in streams if s.streamable and s.stream_url), + None, + ) + if not stream: + logger.debug(f"No streamable result for {asin} at codec={codec}") + continue + + ext = _file_extension(stream.codec or codec) + safe = "".join( + ch if ch.isalnum() or ch in " -_." else "_" + for ch in display_name + )[:80] + # T2Tunes always serves audio in an encrypted MP4 container. + # The codec extension (.flac, .opus, .eac3) is only for the + # final decrypted output. + enc_ext = "mp4" if stream.decryption_key else ext + enc_path = self._unique_path(self.download_path / f"{safe}.enc.{enc_ext}") + out_path = self._unique_path(self.download_path / f"{safe}.{ext}") + + if self._engine is not None: + self._engine.update_record( + "amazon", download_id, {"state": "downloading", "progress": 0.0} + ) + try: + downloaded = self._stream_to_file(stream.stream_url, enc_path, download_id) + except Exception as exc: + logger.warning(f"Stream download failed for {asin} ({codec}): {exc}") + enc_path.unlink(missing_ok=True) + continue + + if downloaded < MIN_AUDIO_BYTES: + logger.warning( + f"File too small ({downloaded} B) for {asin} at {codec} — trying next" + ) + enc_path.unlink(missing_ok=True) + continue + + if stream.decryption_key: + if self._engine is not None: + self._engine.update_record( + "amazon", download_id, {"state": "decrypting", "progress": 1.0} + ) + try: + self._decrypt_with_ffmpeg(enc_path, out_path, stream.decryption_key) + enc_path.unlink(missing_ok=True) + except Exception as exc: + logger.error(f"Decryption failed for {asin} ({codec}): {exc}") + enc_path.unlink(missing_ok=True) + out_path.unlink(missing_ok=True) + continue + else: + enc_path.rename(out_path) + + final_size = out_path.stat().st_size + logger.info( + f"Amazon download complete ({codec}): {out_path} " + f"({final_size / (1024 * 1024):.1f} MB)" + ) + # Sync size == transferred so the download monitor's bytes-incomplete + # guard doesn't block post-processing. The throttled updates in + # _stream_to_file leave transferred < size after the last 0.5s tick; + # other streaming clients avoid this by not tracking bytes at all + # (size stays 0, the guard is skipped). Writing the final output size + # here restores parity. + if self._engine is not None: + self._engine.update_record( + "amazon", download_id, {'size': final_size, 'transferred': final_size} + ) + return str(out_path) + + logger.error(f"All codec tiers exhausted for '{display_name}' ({asin})") + return None + + def _decrypt_with_ffmpeg( + self, enc_path: Path, out_path: Path, hex_key: str + ) -> None: + """Decrypt a T2Tunes encrypted audio file using FFmpeg -decryption_key. + + T2Tunes uses CENC (Common Encryption) for DRM-protected tracks. + FFmpeg supports decryption via the -decryption_key flag when the + key is provided in hex. + """ + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + tools_dir = Path(__file__).parent.parent / "tools" + candidate = tools_dir / ("ffmpeg.exe" if os.name == "nt" else "ffmpeg") + if candidate.exists(): + ffmpeg = str(candidate) + else: + raise RuntimeError( + "ffmpeg is required for Amazon Music decryption. " + "Install ffmpeg and ensure it is on your PATH." + ) + cmd = [ + ffmpeg, + "-y", + "-hide_banner", + "-loglevel", "error", + "-decryption_key", hex_key, + "-i", str(enc_path), + "-map", "0:a:0", # extract first audio stream (FLAC/Opus/EAC3 inside MP4) + "-c", "copy", + str(out_path), + ] + logger.debug(f"Decrypting {enc_path.name} → {out_path.name}") + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(f"FFmpeg decryption failed (exit {result.returncode}): {stderr}") + + def _stream_to_file(self, url: str, out_path: Path, download_id: str) -> int: + resp = self.session.get(url, stream=True, timeout=60) + resp.raise_for_status() + + total = int(resp.headers.get("content-length", 0)) + downloaded = 0 + last_report = time.monotonic() + shutdown_triggered = False + + with out_path.open("wb") as fh: + for chunk in resp.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + if self.shutdown_check and self.shutdown_check(): + shutdown_triggered = True + break + fh.write(chunk) + downloaded += len(chunk) + now = time.monotonic() + if self._engine and now - last_report >= 0.5: + self._engine.update_record( + "amazon", + download_id, + { + "transferred": downloaded, + "size": total, + "progress": downloaded / total if total else 0.0, + }, + ) + last_report = now + + if shutdown_triggered: + out_path.unlink(missing_ok=True) + raise RuntimeError("Shutdown requested mid-download") + + return downloaded + + # ------------------------------------------------------------------ + # DownloadSourcePlugin — status interface + # ------------------------------------------------------------------ + + async def get_all_downloads(self) -> List[DownloadStatus]: + if self._engine is None: + return [] + return [ + self._record_to_status(record) + for record in self._engine.iter_records_for_source('amazon') + ] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + if self._engine is None: + return None + record = self._engine.get_record('amazon', download_id) + return self._record_to_status(record) if record is not None else None + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + if self._engine is None: + return False + if self._engine.get_record('amazon', download_id) is None: + return False + self._engine.update_record('amazon', download_id, {'state': 'Cancelled'}) + if remove: + self._engine.remove_record('amazon', download_id) + return True + + async def clear_all_completed_downloads(self) -> bool: + if self._engine is None: + return True + terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} + for record in list(self._engine.iter_records_for_source('amazon')): + if record.get('state') in terminal: + self._engine.remove_record('amazon', record['id']) + return True + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + @staticmethod + def _unique_path(path: Path) -> Path: + if not path.exists(): + return path + stem, suffix = path.stem, path.suffix + for i in range(1, 100): + candidate = path.with_name(f"{stem} ({i}){suffix}") + if not candidate.exists(): + return candidate + return path.with_name(f"{stem}_{uuid.uuid4().hex[:8]}{suffix}") + + @staticmethod + def _record_to_status(rec: Dict[str, Any]) -> DownloadStatus: + return DownloadStatus( + id=str(rec.get('id', '')), + filename=str(rec.get('filename', '')), + username='amazon', + state=str(rec.get('state', 'queued')), + progress=float(rec.get('progress', 0.0)), + size=int(rec.get('size', 0)), + transferred=int(rec.get('transferred', 0)), + speed=int(rec.get('speed', 0)), + time_remaining=rec.get('time_remaining'), + file_path=rec.get('file_path'), + ) diff --git a/core/amazon_worker.py b/core/amazon_worker.py new file mode 100644 index 00000000..2ac47ee4 --- /dev/null +++ b/core/amazon_worker.py @@ -0,0 +1,602 @@ +import re +import threading +import time +from difflib import SequenceMatcher +from typing import Optional, Dict, Any +from datetime import datetime, timedelta +from utils.logging_config import get_logger +from database.music_database import MusicDatabase +from core.amazon_client import AmazonClient +from core.worker_utils import interruptible_sleep, set_album_api_track_count +from core.enrichment.manual_match_honoring import honor_stored_match + +logger = get_logger("amazon_worker") + + +class AmazonWorker: + """Background worker for enriching library artists, albums, and tracks with Amazon Music metadata.""" + + def __init__(self, database: MusicDatabase): + self.db = database + self.client = AmazonClient() + self._amazon_schema_checked = False + + self.running = False + self.paused = False + self.should_stop = False + self.thread = None + self._stop_event = threading.Event() + + self.current_item = None + + self.stats = { + 'matched': 0, + 'not_found': 0, + 'pending': 0, + 'errors': 0, + } + + self.retry_days = 30 + self.name_similarity_threshold = 0.80 + + logger.info("Amazon background worker initialized") + + def _ensure_amazon_schema(self, cursor) -> None: + """Ensure upgraded installs have the Amazon enrichment columns. + + MusicDatabase normally runs this migration during startup, but the + worker should still be defensive because it is the code path that + repeatedly queries these columns in the background. + """ + if self._amazon_schema_checked: + return + + table_columns = { + 'artists': ('amazon_id', 'amazon_match_status', 'amazon_last_attempted'), + 'albums': ('amazon_id', 'amazon_match_status', 'amazon_last_attempted'), + 'tracks': ('amazon_id', 'amazon_match_status', 'amazon_last_attempted'), + } + for table, columns in table_columns.items(): + cursor.execute(f"PRAGMA table_info({table})") + existing = {row[1] for row in cursor.fetchall()} + for column in columns: + if column not in existing: + column_type = 'TIMESTAMP' if column == 'amazon_last_attempted' else 'TEXT' + cursor.execute(f"ALTER TABLE {table} ADD COLUMN {column} {column_type}") + + cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_amazon_id ON artists (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_amazon_status ON artists (amazon_match_status)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_albums_amazon_id ON albums (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_albums_amazon_status ON albums (amazon_match_status)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_tracks_amazon_id ON tracks (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_tracks_amazon_status ON tracks (amazon_match_status)") + cursor.connection.commit() + self._amazon_schema_checked = True + + def start(self): + if self.running: + logger.warning("Worker already running") + return + + self.running = True + self.should_stop = False + self._stop_event.clear() + self.thread = threading.Thread(target=self._run, daemon=True) + self.thread.start() + logger.info("Amazon background worker started") + + def stop(self): + if not self.running: + return + + logger.info("Stopping Amazon worker...") + self.should_stop = True + self.running = False + self._stop_event.set() + + if self.thread: + self.thread.join(timeout=1) + + logger.info("Amazon worker stopped") + + def pause(self): + if not self.running: + logger.warning("Worker not running, cannot pause") + return + self.paused = True + logger.info("Amazon worker paused") + + def resume(self): + if not self.running: + logger.warning("Worker not running, start it first") + return + self.paused = False + logger.info("Amazon worker resumed") + + def get_stats(self) -> Dict[str, Any]: + self.stats['pending'] = self._count_pending_items() + progress = self._get_progress_breakdown() + is_actually_running = self.running and (self.thread is not None and self.thread.is_alive()) + is_idle = is_actually_running and not self.paused and self.stats['pending'] == 0 and self.current_item is None + return { + 'enabled': True, + 'running': is_actually_running and not self.paused, + 'paused': self.paused, + 'idle': is_idle, + 'current_item': self.current_item, + 'stats': self.stats.copy(), + 'progress': progress, + } + + def _run(self): + logger.info("Amazon worker thread started") + while not self.should_stop: + try: + if self.paused: + interruptible_sleep(self._stop_event, 1) + continue + + self.current_item = None + item = self._get_next_item() + + if not item: + logger.debug("No pending items, sleeping...") + interruptible_sleep(self._stop_event, 10) + continue + + self.current_item = item + item_id = item.get('id') or item.get('artist_id') or item.get('album_id') + if item_id is None: + logger.warning(f"Skipping {item.get('type', 'unknown')} with NULL id: {item.get('name', '?')}") + continue + + self._process_item(item) + interruptible_sleep(self._stop_event, 2) + + except Exception as e: + logger.error(f"Error in worker loop: {e}") + interruptible_sleep(self._stop_event, 5) + + logger.info("Amazon worker thread finished") + + def _get_next_item(self) -> Optional[Dict[str, Any]]: + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + self._ensure_amazon_schema(cursor) + + # Priority 1: Unattempted artists + cursor.execute(""" + SELECT id, name FROM artists + WHERE amazon_match_status IS NULL AND id IS NOT NULL + ORDER BY id ASC LIMIT 1 + """) + row = cursor.fetchone() + if row: + return {'type': 'artist', 'id': row[0], 'name': row[1]} + + # Priority 2: Unattempted albums + cursor.execute(""" + SELECT a.id, a.title, ar.name AS artist_name, ar.amazon_id AS artist_amazon_id + FROM albums a + JOIN artists ar ON a.artist_id = ar.id + WHERE a.amazon_match_status IS NULL AND a.id IS NOT NULL + ORDER BY a.id ASC LIMIT 1 + """) + row = cursor.fetchone() + if row: + return {'type': 'album', 'id': row[0], 'name': row[1], 'artist': row[2], 'artist_amazon_id': row[3]} + + # Priority 3: Unattempted tracks + cursor.execute(""" + SELECT t.id, t.title, ar.name AS artist_name, ar.amazon_id AS artist_amazon_id + FROM tracks t + JOIN artists ar ON t.artist_id = ar.id + WHERE t.amazon_match_status IS NULL AND t.id IS NOT NULL + ORDER BY t.id ASC LIMIT 1 + """) + row = cursor.fetchone() + if row: + return {'type': 'track', 'id': row[0], 'name': row[1], 'artist': row[2], 'artist_amazon_id': row[3]} + + not_found_cutoff = datetime.now() - timedelta(days=self.retry_days) + + # Priority 4: Retry not_found artists + cursor.execute(""" + SELECT id, name FROM artists + WHERE amazon_match_status = 'not_found' AND amazon_last_attempted < ? + ORDER BY amazon_last_attempted ASC LIMIT 1 + """, (not_found_cutoff,)) + row = cursor.fetchone() + if row: + logger.info(f"Retrying artist '{row[1]}' (last attempted before cutoff)") + return {'type': 'artist', 'id': row[0], 'name': row[1]} + + # Priority 5: Retry not_found albums + cursor.execute(""" + SELECT a.id, a.title, ar.name AS artist_name, ar.amazon_id AS artist_amazon_id + FROM albums a + JOIN artists ar ON a.artist_id = ar.id + WHERE a.amazon_match_status = 'not_found' AND a.amazon_last_attempted < ? + ORDER BY a.amazon_last_attempted ASC LIMIT 1 + """, (not_found_cutoff,)) + row = cursor.fetchone() + if row: + return {'type': 'album', 'id': row[0], 'name': row[1], 'artist': row[2], 'artist_amazon_id': row[3]} + + # Priority 6: Retry not_found tracks + cursor.execute(""" + SELECT t.id, t.title, ar.name AS artist_name, ar.amazon_id AS artist_amazon_id + FROM tracks t + JOIN artists ar ON t.artist_id = ar.id + WHERE t.amazon_match_status = 'not_found' AND t.amazon_last_attempted < ? + ORDER BY t.amazon_last_attempted ASC LIMIT 1 + """, (not_found_cutoff,)) + row = cursor.fetchone() + if row: + return {'type': 'track', 'id': row[0], 'name': row[1], 'artist': row[2], 'artist_amazon_id': row[3]} + + return None + + except Exception as e: + logger.error(f"Error getting next item: {e}") + return None + finally: + if conn: + conn.close() + + def _normalize_name(self, name: str) -> str: + name = name.lower().strip() + name = re.sub(r'\s+[-–—]\s+.*$', '', name) + name = re.sub(r'\s*\(.*?\)\s*', ' ', name) + name = re.sub(r'[^\w\s]', '', name) + name = re.sub(r'\s+', ' ', name).strip() + return name + + def _name_matches(self, query_name: str, result_name: str) -> bool: + norm_query = self._normalize_name(query_name) + norm_result = self._normalize_name(result_name) + similarity = SequenceMatcher(None, norm_query, norm_result).ratio() + logger.debug(f"Name similarity: '{query_name}' vs '{result_name}' = {similarity:.2f}") + return similarity >= self.name_similarity_threshold + + def _process_item(self, item: Dict[str, Any]): + try: + item_type = item['type'] + item_id = item['id'] + item_name = item['name'] + logger.debug(f"Processing {item_type} #{item_id}: {item_name}") + + if item_type == 'artist': + self._process_artist(item_id, item_name) + elif item_type == 'album': + self._process_album(item_id, item_name, item.get('artist', ''), item) + elif item_type == 'track': + self._process_track(item_id, item_name, item.get('artist', ''), item) + + except Exception as e: + logger.error(f"Error processing {item['type']} #{item['id']}: {e}") + self.stats['errors'] += 1 + try: + self._mark_status(item['type'], item['id'], 'error') + except Exception as e2: + logger.error(f"Error updating item status: {e2}") + + def _get_existing_id(self, entity_type: str, entity_id: int) -> Optional[str]: + table_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'} + table = table_map.get(entity_type) + if not table: + return None + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cursor.execute(f"SELECT amazon_id FROM {table} WHERE id = ?", (entity_id,)) + row = cursor.fetchone() + return row[0] if row and row[0] else None + except Exception: + return None + finally: + if conn: + conn.close() + + def _process_artist(self, artist_id: int, artist_name: str): + existing_id = self._get_existing_id('artist', artist_id) + if existing_id: + logger.debug(f"Preserving existing Amazon ID for artist '{artist_name}': {existing_id}") + return + + results = self.client.search_artists(artist_name, limit=5) + if results: + result = results[0] + if self._name_matches(artist_name, result.name): + self._update_artist(artist_id, result) + self.stats['matched'] += 1 + logger.info(f"Matched artist '{artist_name}' -> Amazon ID: {result.id}") + else: + self._mark_status('artist', artist_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"Name mismatch for artist '{artist_name}' (got '{result.name}')") + else: + self._mark_status('artist', artist_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"No match for artist '{artist_name}'") + + def _refresh_album_via_stored_id(self, album_id, stored_id, api_data): + self._update_album(album_id, api_data, stored_id) + + def _refresh_track_via_stored_id(self, track_id, stored_id, api_data): + self._update_track(track_id, api_data, stored_id) + + def _process_album(self, album_id: int, album_name: str, artist_name: str, item: Dict[str, Any]): + if honor_stored_match( + db=self.db, entity_table='albums', entity_id=album_id, + id_column='amazon_id', + client_fetch_fn=lambda asin: self.client.get_album(asin, include_tracks=False), + on_match_fn=self._refresh_album_via_stored_id, + log_prefix='Amazon', + ): + self.stats['matched'] += 1 + return + + query = f"{artist_name} {album_name}" + results = self.client.search_albums(query, limit=10) + if results: + result = results[0] + if self._name_matches(album_name, result.name): + full_album = None + if result.id: + try: + full_album = self.client.get_album(result.id, include_tracks=False) + except Exception as e: + logger.warning(f"Failed to fetch full album '{album_name}' (ASIN: {result.id}): {e}") + + if full_album is None: + self._mark_status('album', album_id, 'error') + self.stats['errors'] += 1 + logger.warning(f"Album '{album_name}' matched but full details unavailable, will retry") + return + + self._update_album(album_id, full_album, result.id) + self.stats['matched'] += 1 + logger.info(f"Matched album '{album_name}' -> Amazon ASIN: {result.id}") + else: + self._mark_status('album', album_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"Name mismatch for album '{album_name}' (got '{result.name}')") + else: + self._mark_status('album', album_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"No match for album '{album_name}'") + + def _process_track(self, track_id: int, track_name: str, artist_name: str, item: Dict[str, Any]): + if honor_stored_match( + db=self.db, entity_table='tracks', entity_id=track_id, + id_column='amazon_id', + client_fetch_fn=self.client.get_track_details, + on_match_fn=self._refresh_track_via_stored_id, + log_prefix='Amazon', + ): + self.stats['matched'] += 1 + return + + query = f"{artist_name} {track_name}" + results = self.client.search_tracks(query, limit=10) + if results: + result = results[0] + if self._name_matches(track_name, result.name): + full_track = None + if result.id: + try: + full_track = self.client.get_track_details(result.id) + except Exception as e: + logger.warning(f"Failed to fetch full track '{track_name}' (ASIN: {result.id}): {e}") + + if full_track is None: + self._mark_status('track', track_id, 'error') + self.stats['errors'] += 1 + logger.warning(f"Track '{track_name}' matched but full details unavailable, will retry") + return + + self._update_track(track_id, full_track, result.id) + self.stats['matched'] += 1 + logger.info(f"Matched track '{track_name}' -> Amazon ASIN: {result.id}") + else: + self._mark_status('track', track_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"Name mismatch for track '{track_name}' (got '{result.name}')") + else: + self._mark_status('track', track_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"No match for track '{track_name}'") + + def _update_artist(self, artist_id: int, result): + """Store Amazon metadata for an artist. ``result`` is an Artist dataclass.""" + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + + cursor.execute(""" + UPDATE artists SET + amazon_id = ?, + amazon_match_status = 'matched', + amazon_last_attempted = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (str(result.id), artist_id)) + + # Backfill thumb_url from album cover stand-in when artist has no image + image_url = result.image_url + if not image_url: + try: + image_url = self.client._get_artist_image_from_albums(result.id) + except Exception as exc: + logger.debug("Artist image from albums failed for %s: %s", result.id, exc) + if image_url: + cursor.execute(""" + UPDATE artists SET thumb_url = ? + WHERE id = ? AND (thumb_url IS NULL OR thumb_url = '') + """, (image_url, artist_id)) + + conn.commit() + + except Exception as e: + logger.error(f"Error updating artist #{artist_id} with Amazon data: {e}") + raise + finally: + if conn: + conn.close() + + def _update_album(self, album_id: int, full_data: Dict[str, Any], asin: str): + """Store Amazon metadata for an album. ``full_data`` is a get_album() dict.""" + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + + cursor.execute(""" + UPDATE albums SET + amazon_id = ?, + amazon_match_status = 'matched', + amazon_last_attempted = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (asin, album_id)) + + # Backfill label when missing + label = full_data.get('label') + if label: + cursor.execute(""" + UPDATE albums SET label = ? + WHERE id = ? AND (label IS NULL OR label = '') + """, (label, album_id)) + + # Backfill thumb_url + images = full_data.get('images') or [] + thumb_url = images[0].get('url') if images else None + if thumb_url: + cursor.execute(""" + UPDATE albums SET thumb_url = ? + WHERE id = ? AND (thumb_url IS NULL OR thumb_url = '') + """, (thumb_url, album_id)) + + # Cache authoritative track count for completeness repair + total_tracks = full_data.get('total_tracks') or ( + full_data.get('tracks', {}).get('total') if isinstance(full_data.get('tracks'), dict) else None + ) + set_album_api_track_count(cursor, album_id, total_tracks) + + conn.commit() + + except Exception as e: + logger.error(f"Error updating album #{album_id} with Amazon data: {e}") + raise + finally: + if conn: + conn.close() + + def _update_track(self, track_id: int, full_data: Dict[str, Any], asin: str): + """Store Amazon metadata for a track. ``full_data`` is a get_track_details() dict.""" + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + + cursor.execute(""" + UPDATE tracks SET + amazon_id = ?, + amazon_match_status = 'matched', + amazon_last_attempted = CURRENT_TIMESTAMP + WHERE id = ? + """, (asin, track_id)) + + conn.commit() + + except Exception as e: + logger.error(f"Error updating track #{track_id} with Amazon data: {e}") + raise + finally: + if conn: + conn.close() + + def _mark_status(self, entity_type: str, entity_id: int, status: str): + table_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'} + table = table_map.get(entity_type) + if not table: + logger.error(f"Unknown entity type: {entity_type}") + return + + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cursor.execute(f""" + UPDATE {table} SET + amazon_match_status = ?, + amazon_last_attempted = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (status, entity_id)) + conn.commit() + except Exception as e: + logger.error(f"Error marking {entity_type} #{entity_id} status: {e}") + finally: + if conn: + conn.close() + + def _count_pending_items(self) -> int: + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + self._ensure_amazon_schema(cursor) + cursor.execute(""" + SELECT + (SELECT COUNT(*) FROM artists WHERE amazon_match_status IS NULL AND id IS NOT NULL) + + (SELECT COUNT(*) FROM albums WHERE amazon_match_status IS NULL AND id IS NOT NULL) + + (SELECT COUNT(*) FROM tracks WHERE amazon_match_status IS NULL AND id IS NOT NULL) + AS pending + """) + row = cursor.fetchone() + return row[0] if row else 0 + except Exception as e: + logger.error(f"Error counting pending items: {e}") + return 0 + finally: + if conn: + conn.close() + + def _get_progress_breakdown(self) -> Dict[str, Dict[str, int]]: + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + self._ensure_amazon_schema(cursor) + progress = {} + + for table in ('artists', 'albums', 'tracks'): + cursor.execute(f""" + SELECT + COUNT(*) AS total, + SUM(CASE WHEN amazon_match_status IS NOT NULL THEN 1 ELSE 0 END) AS processed + FROM {table} + """) + row = cursor.fetchone() + if row: + total, processed = row[0], row[1] or 0 + progress[table] = { + 'matched': processed, + 'total': total, + 'percent': int((processed / total * 100) if total > 0 else 0), + } + + return progress + + except Exception as e: + logger.error(f"Error getting progress breakdown: {e}") + return {} + finally: + if conn: + conn.close() diff --git a/core/api_call_tracker.py b/core/api_call_tracker.py index 7fb561e1..95e1e18e 100644 --- a/core/api_call_tracker.py +++ b/core/api_call_tracker.py @@ -30,6 +30,7 @@ RATE_LIMITS = { 'tidal': 120, # MIN_API_INTERVAL=0.5s → ~120/min 'qobuz': 60, # Variable throttle, ~60/min estimate 'discogs': 60, # MIN_API_INTERVAL=1.0s with auth → ~60/min + 'amazon': 120, # MIN_API_INTERVAL=0.5s → ~120/min (T2Tunes proxy) } # Display names for UI @@ -44,12 +45,13 @@ SERVICE_LABELS = { 'tidal': 'Tidal', 'qobuz': 'Qobuz', 'discogs': 'Discogs', + 'amazon': 'Amazon Music', } # Display order SERVICE_ORDER = [ 'spotify', 'itunes', 'deezer', 'lastfm', 'genius', - 'musicbrainz', 'audiodb', 'tidal', 'qobuz', 'discogs', + 'musicbrainz', 'audiodb', 'tidal', 'qobuz', 'discogs', 'amazon', ] diff --git a/core/archive_pipeline.py b/core/archive_pipeline.py new file mode 100644 index 00000000..138e520c --- /dev/null +++ b/core/archive_pipeline.py @@ -0,0 +1,238 @@ +"""Archive extraction + audio-file discovery for torrent / usenet downloads. + +The torrent and usenet download plugins need a uniform way to: + +1. Walk the downloader's save directory and find every audio file in it. +2. If the directory contains an archive (``.zip`` / ``.rar`` / ``.tar`` / + ``.7z``), extract it first so the audio files inside become walkable. + +This module is intentionally narrow — no matching, no tagging, no +import. The download plugin layer composes this with the existing +post-processing / matching pipeline. Lidarr does NOT use this module: +Lidarr extracts archives in its own import step before SoulSync sees +the files at all. Usenet downloaders (SABnzbd, NZBGet) also auto- +extract by default. Torrents are the main case where SoulSync may +need to do the extract step itself — most music torrents ship loose, +but some bundle the album in a ``.rar`` archive. + +``rarfile`` is an optional dependency. If it isn't installed, archives +with ``.rar`` content are skipped with a single warning rather than +crashing the download. +""" + +from __future__ import annotations + +import tarfile +import zipfile +from pathlib import Path +from typing import List, Optional + +from utils.logging_config import get_logger + +logger = get_logger("archive_pipeline") + + +# Same audio-extension set as ``core/imports/file_ops.py`` ``quality_tiers``. +# Keep them in sync — if a new format is added to file_ops, add it here too +# or the walker will skip it and the download plugin will mark the download +# failed even when files arrived. +AUDIO_EXTENSIONS = frozenset([ + # lossless + '.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif', + # high lossy + '.opus', '.ogg', + # standard lossy + '.m4a', '.aac', + # low lossy + '.mp3', '.wma', +]) + +ARCHIVE_EXTENSIONS = frozenset(['.zip', '.rar', '.tar', '.tar.gz', '.tgz', '.7z']) + + +def is_archive(path: Path) -> bool: + """True if the file extension looks like a supported archive. + + Compound extensions (``.tar.gz``, ``.tar.bz2``) are detected by + checking the last two suffixes joined together — Path.suffix + only returns the final suffix. + """ + if not path.is_file(): + return False + name = path.name.lower() + if name.endswith(('.tar.gz', '.tar.bz2', '.tar.xz')): + return True + return path.suffix.lower() in ARCHIVE_EXTENSIONS + + +def walk_audio_files(directory: Path) -> List[Path]: + """Recursively scan ``directory`` for audio files. Returns + a sorted list of absolute paths. Empty list if the directory + doesn't exist or contains no audio. + """ + if not directory or not directory.exists() or not directory.is_dir(): + return [] + out: List[Path] = [] + for child in directory.rglob('*'): + if not child.is_file(): + continue + if child.suffix.lower() in AUDIO_EXTENSIONS: + out.append(child.resolve()) + out.sort() + return out + + +def find_archives_in_dir(directory: Path) -> List[Path]: + """Find every archive file directly inside ``directory`` (one + level deep — torrents normally put the archive at the root of + their folder; we don't search nested dirs to avoid extracting + something we shouldn't). + """ + if not directory or not directory.exists() or not directory.is_dir(): + return [] + return sorted(p for p in directory.iterdir() if is_archive(p)) + + +def extract_archive(archive_path: Path, extract_to: Optional[Path] = None) -> Optional[Path]: + """Extract a single archive in-place (or to ``extract_to`` if + given). Returns the directory the archive was extracted into, + or ``None`` on failure. + + Supports ``.zip``, ``.tar``/``.tar.gz``/``.tar.bz2``/``.tar.xz``, + and ``.rar`` (only when the optional ``rarfile`` library is + installed). ``.7z`` is recognised but extraction requires + ``py7zr``; without it, the call logs and returns None. + """ + if not archive_path or not archive_path.exists(): + logger.warning("archive_pipeline: %s does not exist", archive_path) + return None + dest = extract_to or archive_path.parent + dest.mkdir(parents=True, exist_ok=True) + + name = archive_path.name.lower() + try: + if name.endswith('.zip'): + with zipfile.ZipFile(archive_path) as zf: + _safe_extract_zip(zf, dest) + return dest + if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz')): + with tarfile.open(archive_path) as tf: + _safe_extract_tar(tf, dest) + return dest + if name.endswith('.rar'): + return _extract_rar(archive_path, dest) + if name.endswith('.7z'): + return _extract_7z(archive_path, dest) + except (zipfile.BadZipFile, tarfile.TarError, OSError) as e: + logger.error("archive_pipeline: failed to extract %s: %s", archive_path, e) + return None + logger.warning("archive_pipeline: unknown archive type for %s", archive_path) + return None + + +def extract_all_in_dir(directory: Path) -> List[Path]: + """Find every archive in ``directory`` and extract each in place. + Returns the list of directories archives were extracted into + (usually all the same — ``directory`` itself). Archives that + failed to extract are skipped silently after a warning. + """ + out: List[Path] = [] + for archive in find_archives_in_dir(directory): + result = extract_archive(archive) + if result is not None: + out.append(result) + return out + + +def collect_audio_after_extraction(directory: Path) -> List[Path]: + """One-shot helper for the download plugins: extract any archives + in the directory, then return the walked audio file list. This is + the common pattern — torrent / usenet plugin gets a save_path, + calls this, hands the resulting files to the matching pipeline. + """ + extract_all_in_dir(directory) + return walk_audio_files(directory) + + +# --------------------------------------------------------------------------- +# Safety helpers +# --------------------------------------------------------------------------- + + +def _safe_extract_zip(zf: zipfile.ZipFile, dest: Path) -> None: + """Extract a zipfile after rejecting any member whose resolved + path escapes ``dest`` (path traversal protection). + """ + dest = dest.resolve() + for member in zf.namelist(): + target = (dest / member).resolve() + if dest not in target.parents and target != dest: + logger.error("archive_pipeline: refusing path-traversal member %r", member) + return + zf.extractall(dest) + + +def _safe_extract_tar(tf: tarfile.TarFile, dest: Path) -> None: + """Same path-traversal protection for tarfiles.""" + dest = dest.resolve() + for member in tf.getmembers(): + target = (dest / member.name).resolve() + if dest not in target.parents and target != dest: + logger.error("archive_pipeline: refusing path-traversal member %r", member.name) + return + # ``filter='data'`` is the Python 3.12+ safe extractor; fall back + # to the legacy call on older runtimes. + try: + tf.extractall(dest, filter='data') # type: ignore[call-arg] + except TypeError: + tf.extractall(dest) + + +def _extract_rar(archive_path: Path, dest: Path) -> Optional[Path]: + try: + import rarfile # type: ignore[import-untyped] + except ImportError: + logger.warning( + "archive_pipeline: cannot extract %s — rarfile library not installed. " + "Install with: pip install rarfile (and ensure unrar is on PATH).", + archive_path, + ) + return None + try: + with rarfile.RarFile(archive_path) as rf: + dest_resolved = dest.resolve() + for name in rf.namelist(): + target = (dest_resolved / name).resolve() + if dest_resolved not in target.parents and target != dest_resolved: + logger.error("archive_pipeline: refusing path-traversal rar member %r", name) + return None + rf.extractall(dest) + return dest + except Exception as e: + logger.error("archive_pipeline: rar extract failed for %s: %s", archive_path, e) + return None + + +def _extract_7z(archive_path: Path, dest: Path) -> Optional[Path]: + try: + import py7zr # type: ignore[import-untyped] + except ImportError: + logger.warning( + "archive_pipeline: cannot extract %s — py7zr library not installed. " + "Install with: pip install py7zr.", + archive_path, + ) + return None + try: + with py7zr.SevenZipFile(archive_path, 'r') as sz: + dest_resolved = dest.resolve() + for name in sz.getnames(): + target = (dest_resolved / name).resolve() + if dest_resolved not in target.parents and target != dest_resolved: + logger.error("archive_pipeline: refusing path-traversal 7z member %r", name) + return None + sz.extractall(path=dest) + return dest + except Exception as e: + logger.error("archive_pipeline: 7z extract failed for %s: %s", archive_path, e) + return None diff --git a/core/artist_source_detail.py b/core/artist_source_detail.py index c376052e..5576f224 100644 --- a/core/artist_source_detail.py +++ b/core/artist_source_detail.py @@ -43,6 +43,7 @@ def build_source_only_artist_detail( deezer_client: Optional[Any] = None, itunes_client: Optional[Any] = None, discogs_client: Optional[Any] = None, + amazon_client: Optional[Any] = None, lastfm_api_key: Optional[str] = None, ) -> Tuple[Dict[str, Any], int]: """Build the artist-detail payload for a source-only artist. @@ -51,7 +52,7 @@ def build_source_only_artist_detail( ``jsonify`` or equivalent. Status is 200 on success, 404 when the source's discography lookup returned no releases. """ - resolved_name = (artist_name or artist_id or "").strip() + resolved_name = (artist_name or "").strip() # 1. Image URL via the same helper /api/artist//image uses. image_url: Optional[str] = None @@ -67,6 +68,8 @@ def build_source_only_artist_detail( if source == "spotify" and spotify_client is not None: sp_artist = spotify_client.get_artist(artist_id, allow_fallback=False) if sp_artist: + if not artist_name and sp_artist.get("name"): + resolved_name = sp_artist["name"] source_genres = sp_artist.get("genres") or [] source_followers = (sp_artist.get("followers") or {}).get("total") if not image_url and sp_artist.get("images"): @@ -74,16 +77,41 @@ def build_source_only_artist_detail( elif source == "deezer" and deezer_client is not None: dz_artist = deezer_client.get_artist_info(artist_id) if dz_artist: + if not artist_name and dz_artist.get("name"): + resolved_name = dz_artist["name"] source_genres = dz_artist.get("genres") or [] source_followers = (dz_artist.get("followers") or {}).get("total") elif source == "itunes" and itunes_client is not None: it_artist = itunes_client.get_artist(artist_id) if it_artist: + if not artist_name and it_artist.get("name"): + resolved_name = it_artist["name"] source_genres = it_artist.get("genres") or [] elif source == "discogs" and discogs_client is not None: dc_artist = discogs_client.get_artist(artist_id) if dc_artist: + if not artist_name and dc_artist.get("name"): + resolved_name = dc_artist["name"] source_genres = dc_artist.get("genres") or [] + elif source == "amazon" and amazon_client is not None: + az_artist = amazon_client.get_artist(resolved_name or artist_id) + if az_artist: + if not artist_name and az_artist.get("name"): + resolved_name = az_artist["name"] + source_genres = az_artist.get("genres") or [] + if not image_url and az_artist.get("images"): + image_url = az_artist["images"][0].get("url") + elif source == "musicbrainz": + try: + from core.musicbrainz_search import MusicBrainzSearchClient + mb = MusicBrainzSearchClient() + mb_artist = mb.get_artist(artist_id) + if mb_artist: + if not artist_name and mb_artist.get("name"): + resolved_name = mb_artist["name"] + source_genres = mb_artist.get("genres") or [] + except Exception as e: + logger.debug(f"MusicBrainz artist info lookup failed for {artist_id}: {e}") except Exception as e: logger.debug(f"Source-side artist info lookup failed for {source}:{artist_id}: {e}") diff --git a/core/artist_source_lookup.py b/core/artist_source_lookup.py index b965d4cf..507d09e6 100644 --- a/core/artist_source_lookup.py +++ b/core/artist_source_lookup.py @@ -25,7 +25,7 @@ logger = logging.getLogger("artist_source_lookup") SOURCE_ONLY_ARTIST_SOURCES = frozenset({ - "spotify", "itunes", "deezer", "discogs", "hydrabase", "musicbrainz", + "spotify", "itunes", "deezer", "discogs", "hydrabase", "musicbrainz", "amazon", }) @@ -36,6 +36,7 @@ SOURCE_ID_FIELD = { "discogs": "discogs_id", "hydrabase": "soul_id", "musicbrainz": "musicbrainz_id", + "amazon": "amazon_id", } diff --git a/core/artists/liked_match.py b/core/artists/liked_match.py index 324d8cbc..a8a973ab 100644 --- a/core/artists/liked_match.py +++ b/core/artists/liked_match.py @@ -12,6 +12,7 @@ from core.metadata.registry import ( get_deezer_client, get_discogs_client, get_itunes_client, + get_musicbrainz_client, get_spotify_client, ) @@ -33,6 +34,11 @@ def _get_discogs_client(token=None): return get_discogs_client(token) +def _get_musicbrainz_client(): + """Mirror of web_server._get_musicbrainz_client — delegates to registry.""" + return get_musicbrainz_client() + + class _SpotifyClientProxy: """Resolves the global Spotify client lazily so a Spotify re-auth that rebinds the cached client in core.metadata.registry is visible to the @@ -65,6 +71,7 @@ def _match_liked_artists_to_all_sources(database, profile_id: int): 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id', + 'musicbrainz': 'musicbrainz_artist_id', } id_cols = list(source_cols.values()) @@ -103,6 +110,10 @@ def _match_liked_artists_to_all_sources(database, profile_id: int): search_clients['discogs'] = dc except Exception as e: logger.debug("discogs client init failed: %s", e) + try: + search_clients['musicbrainz'] = _get_musicbrainz_client() + except Exception as e: + logger.debug("musicbrainz client init failed: %s", e) # Reuse watchlist scanner's fuzzy matching logic from core.watchlist_scanner import WatchlistScanner @@ -234,7 +245,7 @@ def _match_liked_artists_to_all_sources(database, profile_id: int): # Determine best active source/ID — prefer Spotify, then iTunes, Deezer, Discogs resolved_source = None resolved_id = None - for src in ('spotify', 'itunes', 'deezer', 'discogs'): + for src in ('spotify', 'itunes', 'deezer', 'discogs', 'musicbrainz'): col = source_cols[src] if col in harvested_ids: resolved_source = src diff --git a/core/artists/map.py b/core/artists/map.py index 5e6f130a..781a8d1e 100644 --- a/core/artists/map.py +++ b/core/artists/map.py @@ -20,10 +20,14 @@ logger = logging.getLogger(__name__) def get_current_profile_id() -> int: - """Mirror of web_server.get_current_profile_id — uses Flask g.""" + """Mirror of web_server.get_current_profile_id — uses Flask g. + + Catches RuntimeError too because reading `g` outside a request + context raises that (not AttributeError) — happens when this is + called from background threads (sync, automation, scanners).""" try: return g.profile_id - except AttributeError: + except (AttributeError, RuntimeError): return 1 @@ -136,7 +140,7 @@ def get_artist_map_data(): placeholders = ','.join(['?'] * len(watchlist_ids)) cursor.execute(f""" SELECT source_artist_id, similar_artist_name, similar_artist_spotify_id, - similar_artist_itunes_id, similar_artist_deezer_id, + similar_artist_itunes_id, similar_artist_deezer_id, similar_artist_musicbrainz_id, similarity_rank, occurrence_count, image_url, genres, popularity FROM similar_artists WHERE profile_id = ? AND source_artist_id IN ({placeholders}) @@ -169,6 +173,7 @@ def get_artist_map_data(): 'spotify_id': r.get('similar_artist_spotify_id') or '', 'itunes_id': r.get('similar_artist_itunes_id') or '', 'deezer_id': r.get('similar_artist_deezer_id') or '', + 'musicbrainz_id': r.get('similar_artist_musicbrainz_id') or '', 'rank': r.get('similarity_rank', 5), 'occurrence': r.get('occurrence_count', 1), 'popularity': r.get('popularity', 0), @@ -241,7 +246,7 @@ def get_artist_map_data(): } # Apply cache data to nodes - source_id_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} + source_id_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id', 'musicbrainz': 'musicbrainz_id'} for n in nodes: nn = _norm(n['name']) cached = cache_by_name.get(nn) @@ -258,7 +263,7 @@ def get_artist_map_data(): break # Backfill genres if missing if not n.get('genres') or len(n.get('genres', [])) == 0: - for source in ('spotify', 'deezer', 'itunes', 'discogs'): + for source in ('spotify', 'deezer', 'itunes', 'discogs', 'musicbrainz'): if source in cached and cached[source].get('genres'): n['genres'] = cached[source]['genres'][:5] break @@ -365,14 +370,14 @@ def get_artist_map_genres(): def _norm(n): return (n or '').lower().strip() - def _add(name, image_url=None, genres=None, spotify_id=None, itunes_id=None, deezer_id=None, discogs_id=None, source='unknown', popularity=0): + def _add(name, image_url=None, genres=None, spotify_id=None, itunes_id=None, deezer_id=None, discogs_id=None, musicbrainz_id=None, source='unknown', popularity=0): n = _norm(name) if not n or len(n) < 2: return if n not in artists_by_name: artists_by_name[n] = { 'name': name, 'image_url': '', 'genres': set(), - 'spotify_id': '', 'itunes_id': '', 'deezer_id': '', 'discogs_id': '', + 'spotify_id': '', 'itunes_id': '', 'deezer_id': '', 'discogs_id': '', 'musicbrainz_id': '', 'sources': set(), 'popularity': 0 } a = artists_by_name[n] @@ -390,6 +395,8 @@ def get_artist_map_genres(): a['deezer_id'] = str(deezer_id) if discogs_id and not a['discogs_id']: a['discogs_id'] = str(discogs_id) + if musicbrainz_id and not a['musicbrainz_id']: + a['musicbrainz_id'] = str(musicbrainz_id) if popularity > a['popularity']: a['popularity'] = popularity a['sources'].add(source) @@ -406,14 +413,14 @@ def get_artist_map_genres(): genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] except Exception as e: logger.debug("cache artist genres parse failed: %s", e) - src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} + src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id', 'musicbrainz': 'musicbrainz_id'} kwargs = {src_map.get(r['source'], 'spotify_id'): r['entity_id']} _add(r['name'], image_url=r['image_url'], genres=genres, source='cache', popularity=r['popularity'] or 0, **kwargs) # 2. Similar artists cursor.execute(""" SELECT similar_artist_name, similar_artist_spotify_id, similar_artist_itunes_id, - similar_artist_deezer_id, image_url, genres, popularity + similar_artist_deezer_id, similar_artist_musicbrainz_id, image_url, genres, popularity FROM similar_artists WHERE profile_id = ? """, (profile_id,)) for r in cursor.fetchall(): @@ -425,7 +432,9 @@ def get_artist_map_genres(): logger.debug("similar artist genres parse failed: %s", e) _add(r['similar_artist_name'], image_url=r['image_url'], genres=genres, spotify_id=r['similar_artist_spotify_id'], itunes_id=r['similar_artist_itunes_id'], - deezer_id=r['similar_artist_deezer_id'], source='similar', popularity=r['popularity'] or 0) + deezer_id=r['similar_artist_deezer_id'], + musicbrainz_id=r['similar_artist_musicbrainz_id'] if 'similar_artist_musicbrainz_id' in r.keys() else None, + source='similar', popularity=r['popularity'] or 0) # 3. Watchlist artists cursor.execute(""" @@ -479,6 +488,7 @@ def get_artist_map_genres(): 'itunes_id': a['itunes_id'], 'deezer_id': a['deezer_id'], 'discogs_id': a['discogs_id'], + 'musicbrainz_id': a['musicbrainz_id'], 'popularity': a['popularity'], 'type': 'watchlist' if 'watchlist' in a['sources'] else 'similar', }) @@ -622,7 +632,7 @@ def get_artist_map_explore(): # Find the center artist center_name = artist_name center_image = '' - center_ids = {'spotify_id': '', 'itunes_id': '', 'deezer_id': '', 'discogs_id': ''} + center_ids = {'spotify_id': '', 'itunes_id': '', 'deezer_id': '', 'discogs_id': '', 'musicbrainz_id': ''} center_genres = [] # Search metadata cache for the center artist @@ -644,7 +654,7 @@ def get_artist_map_explore(): center_name = row['name'] if row['image_url'] and row['image_url'].startswith('http'): center_image = row['image_url'] - src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} + src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id', 'musicbrainz': 'musicbrainz_id'} k = src_map.get(row['source'], 'spotify_id') center_ids[k] = row['entity_id'] if row['genres']: @@ -655,14 +665,14 @@ def get_artist_map_explore(): # Check watchlist + library if not in cache if not artist_found and not artist_id: - cursor.execute("SELECT artist_name, image_url, spotify_artist_id, itunes_artist_id, deezer_artist_id, discogs_artist_id FROM watchlist_artists WHERE artist_name = ? COLLATE NOCASE LIMIT 1", (artist_name,)) + cursor.execute("SELECT artist_name, image_url, spotify_artist_id, itunes_artist_id, deezer_artist_id, discogs_artist_id, musicbrainz_artist_id FROM watchlist_artists WHERE artist_name = ? COLLATE NOCASE LIMIT 1", (artist_name,)) wr = cursor.fetchone() if wr: artist_found = True center_name = wr['artist_name'] if wr['image_url'] and str(wr['image_url']).startswith('http'): center_image = wr['image_url'] - for k, col in [('spotify_id', 'spotify_artist_id'), ('itunes_id', 'itunes_artist_id'), ('deezer_id', 'deezer_artist_id'), ('discogs_id', 'discogs_artist_id')]: + for k, col in [('spotify_id', 'spotify_artist_id'), ('itunes_id', 'itunes_artist_id'), ('deezer_id', 'deezer_artist_id'), ('discogs_id', 'discogs_artist_id'), ('musicbrainz_id', 'musicbrainz_artist_id')]: if wr[col]: center_ids[k] = str(wr[col]) else: @@ -713,7 +723,7 @@ def get_artist_map_explore(): WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE """, (center_name,)) for r in cursor.fetchall(): - src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} + src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id', 'musicbrainz': 'musicbrainz_id'} k = src_map.get(r['source'], 'spotify_id') if not center_ids.get(k): center_ids[k] = r['entity_id'] @@ -742,7 +752,7 @@ def get_artist_map_explore(): placeholders = ','.join(['?'] * len(id_values)) cursor.execute(f""" SELECT DISTINCT similar_artist_name, similar_artist_spotify_id, - similar_artist_itunes_id, similar_artist_deezer_id, + similar_artist_itunes_id, similar_artist_deezer_id, similar_artist_musicbrainz_id, image_url, genres, popularity, similarity_rank FROM similar_artists WHERE source_artist_id IN ({placeholders}) AND profile_id = ? @@ -753,7 +763,7 @@ def get_artist_map_explore(): # Also search by name (the center artist might be a watchlist source) cursor.execute(""" SELECT DISTINCT sa.similar_artist_name, sa.similar_artist_spotify_id, - sa.similar_artist_itunes_id, sa.similar_artist_deezer_id, + sa.similar_artist_itunes_id, sa.similar_artist_deezer_id, sa.similar_artist_musicbrainz_id, sa.image_url, sa.genres, sa.popularity, sa.similarity_rank FROM similar_artists sa JOIN watchlist_artists wa ON sa.source_artist_id = COALESCE(wa.spotify_artist_id, wa.itunes_artist_id, CAST(wa.id AS TEXT)) @@ -785,7 +795,8 @@ def get_artist_map_explore(): image_url=sa.get('image_url'), genres=sa.get('genres'), popularity=sa.get('popularity', 0), - similar_artist_deezer_id=sa.get('deezer_id') + similar_artist_deezer_id=sa.get('deezer_id'), + similar_artist_musicbrainz_id=sa.get('musicbrainz_id'), ) except Exception as e: logger.debug("similar artist insert failed: %s", e) @@ -794,7 +805,7 @@ def get_artist_map_explore(): placeholders = ','.join(['?'] * len(id_values)) cursor.execute(f""" SELECT DISTINCT similar_artist_name, similar_artist_spotify_id, - similar_artist_itunes_id, similar_artist_deezer_id, + similar_artist_itunes_id, similar_artist_deezer_id, similar_artist_musicbrainz_id, image_url, genres, popularity, similarity_rank FROM similar_artists WHERE source_artist_id IN ({placeholders}) AND profile_id = ? @@ -805,7 +816,7 @@ def get_artist_map_explore(): # Fallback: query by name-based source ID cursor.execute(""" SELECT DISTINCT similar_artist_name, similar_artist_spotify_id, - similar_artist_itunes_id, similar_artist_deezer_id, + similar_artist_itunes_id, similar_artist_deezer_id, similar_artist_musicbrainz_id, image_url, genres, popularity, similarity_rank FROM similar_artists WHERE source_artist_id = ? AND profile_id = ? @@ -837,6 +848,7 @@ def get_artist_map_explore(): 'spotify_id': r['similar_artist_spotify_id'] or '', 'itunes_id': r['similar_artist_itunes_id'] or '', 'deezer_id': r['similar_artist_deezer_id'] or '', + 'musicbrainz_id': r['similar_artist_musicbrainz_id'] if 'similar_artist_musicbrainz_id' in r.keys() else '', 'discogs_id': '', 'popularity': r['popularity'] or 0, 'rank': r['similarity_rank'] or 5, @@ -857,7 +869,8 @@ def get_artist_map_explore(): cursor.execute(f""" SELECT DISTINCT source_artist_id, similar_artist_name, similar_artist_spotify_id, similar_artist_itunes_id, - similar_artist_deezer_id, image_url, genres, popularity, similarity_rank + similar_artist_deezer_id, similar_artist_musicbrainz_id, + image_url, genres, popularity, similarity_rank FROM similar_artists WHERE source_artist_id IN ({placeholders}) AND profile_id = ? ORDER BY similarity_rank ASC @@ -898,6 +911,7 @@ def get_artist_map_explore(): 'spotify_id': r['similar_artist_spotify_id'] or '', 'itunes_id': r['similar_artist_itunes_id'] or '', 'deezer_id': r['similar_artist_deezer_id'] or '', + 'musicbrainz_id': r['similar_artist_musicbrainz_id'] if 'similar_artist_musicbrainz_id' in r.keys() else '', 'discogs_id': '', 'popularity': r['popularity'] or 0, 'rank': r['similarity_rank'] or 5, @@ -931,7 +945,7 @@ def get_artist_map_explore(): except Exception as e: logger.debug("explorer node genres parse failed: %s", e) # Harvest missing IDs from cache - src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} + src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id', 'musicbrainz': 'musicbrainz_id'} k = src_map.get(cr['source']) if k and not n.get(k): n[k] = cr['entity_id'] diff --git a/core/automation/__init__.py b/core/automation/__init__.py index 7841a13e..043b8173 100644 --- a/core/automation/__init__.py +++ b/core/automation/__init__.py @@ -1,7 +1,11 @@ -"""Automation API + progress tracking helpers package. +"""Automation API + progress + handlers package. -Lifted from web_server.py /api/automations/* routes and progress -emitters. The action handler registration (`_register_automation_handlers`) -stays in web_server.py because each handler closure is tightly coupled -to other application features. +Lifted from web_server.py: + - `/api/automations/*` route helpers → `api.py` + - block library used by the trigger/action UI → `blocks.py` + - progress tracker (init / update / finish) → `progress.py` + - cross-handler signal bus → `signals.py` + - per-action handler functions → `handlers/` subpackage (with + `deps.py` defining the dependency-injection surface so handlers + stay testable in isolation) """ diff --git a/core/automation/blocks.py b/core/automation/blocks.py index c84d1b37..2697af26 100644 --- a/core/automation/blocks.py +++ b/core/automation/blocks.py @@ -146,6 +146,15 @@ ACTIONS: list[dict] = [ {"key": "all", "type": "checkbox", "label": "Process all mirrored playlists", "default": False}, {"key": "skip_wishlist", "type": "checkbox", "label": "Skip wishlist processing", "default": False}, ]}, + {"type": "personalized_pipeline", "label": "Personalized Playlist Pipeline", "icon": "sparkles", + "description": "Sync personalized / discover-page playlists (Hidden Gems, Time Machine, Fresh Tape, etc.) to your media server + queue missing tracks for download.", + "available": True, + "config_fields": [ + {"key": "kinds", "type": "personalized_playlist_select", "label": "Playlists to sync", + "description": "Multi-select: Hidden Gems, Discovery Shuffle, Time Machine (per decade), Genre playlists, Fresh Tape, The Archives, Seasonal Mix (per season)"}, + {"key": "refresh_first", "type": "checkbox", "label": "Refresh playlists before sync (regenerate snapshots)", "default": False}, + {"key": "skip_wishlist", "type": "checkbox", "label": "Skip wishlist processing", "default": False}, + ]}, {"type": "notify_only", "label": "Notify Only", "icon": "bell", "description": "No action — just send notification", "available": True}, # Phase 3 actions {"type": "start_database_update", "label": "Update Database", "icon": "database", diff --git a/core/automation/deps.py b/core/automation/deps.py new file mode 100644 index 00000000..a521a496 --- /dev/null +++ b/core/automation/deps.py @@ -0,0 +1,146 @@ +"""Dependency-injection surface for automation handlers. + +Each handler in ``core.automation.handlers`` is a top-level pure +function that accepts ``(config: dict, deps: AutomationDeps)`` instead +of reaching for module-level globals in ``web_server``. The deps +namespace bundles every callable, client, and mutable-state container +the handlers need. + +Construction happens once at app startup in ``web_server.py``: + + from core.automation.deps import AutomationDeps, AutomationState + state = AutomationState() + deps = AutomationDeps( + engine=automation_engine, + state=state, + get_database=get_database, + spotify_client=spotify_client, + ... + ) + register_all(deps) + +Tests construct a fake ``AutomationDeps`` with stub callables — every +handler is then exercisable without spinning up Flask, the DB, or +the real media-server clients. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + + +@dataclass +class AutomationState: + """Mutable flags shared across handler invocations. + + Pre-refactor each was a ``global`` or ``nonlocal`` variable inside + the registration closure. Lifted here so handlers + their guards + can read/write a single object instead of importing globals. + + All mutations should hold ``lock``; the helper methods below do + so for the common get/set patterns. + """ + + scan_library_automation_id: Optional[str] = None + db_update_automation_id: Optional[str] = None + pipeline_running: bool = False + lock: threading.Lock = field(default_factory=threading.Lock) + + def is_scan_library_active(self) -> bool: + with self.lock: + return self.scan_library_automation_id is not None + + def is_pipeline_running(self) -> bool: + with self.lock: + return self.pipeline_running + + def set_scan_library_id(self, automation_id: Optional[str]) -> None: + with self.lock: + self.scan_library_automation_id = automation_id + + def set_pipeline_running(self, value: bool) -> None: + with self.lock: + self.pipeline_running = value + + +@dataclass +class AutomationDeps: + """Bundle of every callable + client an automation handler may need. + + Add fields as new handlers are extracted. Every field is required + at construction (no defaults) so a missing dep fails loudly at + startup, not silently mid-handler. + """ + + # --- Engine + shared state --- + engine: Any # AutomationEngine instance + state: AutomationState + config_manager: Any # config.settings.ConfigManager singleton + update_progress: Callable[..., None] # _update_automation_progress + logger: Any # module-level logger from utils.logging_config + + # --- Service clients (each may be None depending on user config) --- + get_database: Callable[[], Any] # late-binding so tests don't need DB + spotify_client: Any + tidal_client: Any + web_scan_manager: Any + + # --- Background-task entry points --- + process_wishlist_automatically: Callable[..., Any] + process_watchlist_scan_automatically: Callable[..., Any] + is_wishlist_actually_processing: Callable[[], bool] + is_watchlist_actually_scanning: Callable[[], bool] + get_watchlist_scan_state: Callable[[], dict] # accessor returns the live mutable dict + + # --- Playlist pipeline entry points --- + run_playlist_discovery_worker: Callable[..., Any] + run_sync_task: Callable[..., Any] + load_sync_status_file: Callable[[], dict] + get_deezer_client: Callable[[], Any] + parse_youtube_playlist: Callable[[str], Any] + get_sync_states: Callable[[], dict] # accessor returns the live dict shared with the sync UI + + # --- Database update + quality scanner (shared state + executors) --- + set_db_update_automation_id: Callable[[Optional[str]], None] # syncs the legacy `_db_update_automation_id` global so the live DB-update progress callbacks (which still read the global directly) keep firing for the active automation + get_db_update_state: Callable[[], dict] + db_update_lock: Any # threading.Lock + db_update_executor: Any # ThreadPoolExecutor + run_db_update_task: Callable[..., Any] + run_deep_scan_task: Callable[..., Any] + get_duplicate_cleaner_state: Callable[[], dict] + duplicate_cleaner_lock: Any + duplicate_cleaner_executor: Any + run_duplicate_cleaner: Callable[..., Any] + get_quality_scanner_state: Callable[[], dict] + quality_scanner_lock: Any + quality_scanner_executor: Any + run_quality_scanner: Callable[..., Any] + + # --- Download orchestrator + queue accessors --- + download_orchestrator: Any + run_async: Callable[..., Any] + tasks_lock: Any + get_download_batches: Callable[[], dict] + get_download_tasks: Callable[[], dict] + sweep_empty_download_directories: Callable[[], int] + get_staging_path: Callable[[], str] + + # --- Maintenance helpers --- + docker_resolve_path: Callable[[str], str] + get_current_profile_id: Callable[[], int] + get_watchlist_scanner: Callable[[Any], Any] + get_app: Callable[[], Any] # Flask app for test_client (beatport refresh) + get_beatport_data_cache: Callable[[], dict] + + # --- Progress + history callbacks (used by register_all to wire + # the engine's progress callback hooks). --- + init_automation_progress: Callable[..., Any] + record_progress_history: Callable[..., Any] + + # --- Personalized playlist pipeline --- + # Lazy builder so the pipeline handler can construct a fresh + # PersonalizedPlaylistManager per run (cheap accessors inside, + # no caching needed yet). + build_personalized_manager: Callable[[], Any] diff --git a/core/automation/handlers/__init__.py b/core/automation/handlers/__init__.py new file mode 100644 index 00000000..7a02f4d4 --- /dev/null +++ b/core/automation/handlers/__init__.py @@ -0,0 +1,64 @@ +"""Per-action automation handlers. + +Each module in this subpackage exposes one top-level handler function +(or a small cluster of related handlers) of the form:: + + def auto_(config: dict, deps: AutomationDeps) -> dict + +The ``register_all`` helper in :mod:`registration` wires every handler +to the engine in one place. ``web_server.py`` calls +``register_all(deps)`` once at startup. +""" + +from core.automation.handlers.process_wishlist import auto_process_wishlist +from core.automation.handlers.scan_watchlist import auto_scan_watchlist +from core.automation.handlers.scan_library import auto_scan_library +from core.automation.handlers.refresh_mirrored import auto_refresh_mirrored +from core.automation.handlers.sync_playlist import auto_sync_playlist +from core.automation.handlers.discover_playlist import auto_discover_playlist +from core.automation.handlers.playlist_pipeline import auto_playlist_pipeline +from core.automation.handlers.personalized_pipeline import auto_personalized_pipeline +from core.automation.handlers.database_update import auto_start_database_update, auto_deep_scan_library +from core.automation.handlers.duplicate_cleaner import auto_run_duplicate_cleaner +from core.automation.handlers.quality_scanner import auto_start_quality_scan +from core.automation.handlers.maintenance import ( + auto_clear_quarantine, + auto_cleanup_wishlist, + auto_update_discovery_pool, + auto_backup_database, + auto_refresh_beatport_cache, +) +from core.automation.handlers.download_cleanup import ( + auto_clean_search_history, + auto_clean_completed_downloads, + auto_full_cleanup, +) +from core.automation.handlers.run_script import auto_run_script +from core.automation.handlers.search_and_download import auto_search_and_download +from core.automation.handlers.registration import register_all + +__all__ = [ + 'auto_process_wishlist', + 'auto_scan_watchlist', + 'auto_scan_library', + 'auto_refresh_mirrored', + 'auto_sync_playlist', + 'auto_discover_playlist', + 'auto_playlist_pipeline', + 'auto_personalized_pipeline', + 'auto_start_database_update', + 'auto_deep_scan_library', + 'auto_run_duplicate_cleaner', + 'auto_start_quality_scan', + 'auto_clear_quarantine', + 'auto_cleanup_wishlist', + 'auto_update_discovery_pool', + 'auto_backup_database', + 'auto_refresh_beatport_cache', + 'auto_clean_search_history', + 'auto_clean_completed_downloads', + 'auto_full_cleanup', + 'auto_run_script', + 'auto_search_and_download', + 'register_all', +] diff --git a/core/automation/handlers/_pipeline_shared.py b/core/automation/handlers/_pipeline_shared.py new file mode 100644 index 00000000..c7427f8a --- /dev/null +++ b/core/automation/handlers/_pipeline_shared.py @@ -0,0 +1,201 @@ +"""Shared helpers between mirrored + personalized playlist pipelines. + +Both pipelines end in the same shape: +1. SYNC each playlist to the active media server. +2. WISHLIST: trigger the wishlist processor for missing tracks. + +The differing prefix (mirrored = REFRESH external sources + DISCOVER +metadata; personalized = SNAPSHOT manager-backed playlists) is owned +by each pipeline. This module owns the SYNC + WISHLIST tail so both +pipelines stay consistent + DRY. +""" + +from __future__ import annotations + +import time +from typing import Any, Callable, Dict, List, Optional + +from core.automation.deps import AutomationDeps + + +# Per-playlist sync poll cap (mirrored side already used this). +_SYNC_PER_PLAYLIST_TIMEOUT_SECONDS = 600 +# Sync-status final-state markers. +_SYNC_TERMINAL_STATUSES = ('finished', 'complete', 'error', 'failed') + + +def run_sync_and_wishlist( + deps: AutomationDeps, + automation_id: Optional[str], + playlists: List[Dict[str, Any]], + *, + sync_one_fn: Callable[[Dict[str, Any]], Dict[str, Any]], + sync_id_for_fn: Callable[[Dict[str, Any]], str], + skip_wishlist: bool = False, + progress_start: int = 56, + progress_end: int = 85, + sync_phase_label: str = 'Phase: Syncing to server...', + sync_phase_start_log: str = 'Sync', + wishlist_phase_label: str = 'Phase: Processing wishlist...', + wishlist_phase_start_log: str = 'Wishlist', +) -> Dict[str, int]: + """Run the SYNC + WISHLIST tail of a playlist pipeline. + + The caller supplies: + - ``playlists``: list of playlist payload dicts. Each must have at + least a ``name`` key (used in progress logs). The shape beyond + ``name`` is opaque to the helper — ``sync_one_fn`` receives the + payload and returns a sync_result dict. + - ``sync_one_fn(payload) -> sync_result``: launches sync for one + playlist. Result dict must carry ``status`` ∈ ``('started', + 'skipped', 'error')`` and may carry ``reason``. + - ``sync_id_for_fn(payload) -> str``: returns the sync-state key + the helper polls on (so we can wait for the background sync + thread to complete + read the matched_tracks count). + + Returns ``{'synced': int, 'skipped': int, 'errors': int, + 'wishlist_queued': int}`` so the caller can stitch it into its + final status. + """ + deps.update_progress( + automation_id, + progress=progress_start, + phase=sync_phase_label, + log_line=sync_phase_start_log, + log_type='info', + ) + + total_synced = 0 + total_skipped = 0 + sync_errors = 0 + sync_states = deps.get_sync_states() + n_playlists = max(1, len(playlists)) + progress_span = max(1, progress_end - progress_start - 1) + + for pl_idx, pl in enumerate(playlists): + pl_name = pl.get('name', '') + sync_result = sync_one_fn(pl) + sync_status = sync_result.get('status', '') + + if sync_status == 'started': + sync_id = sync_id_for_fn(pl) + sync_poll_start = time.time() + while time.time() - sync_poll_start < _SYNC_PER_PLAYLIST_TIMEOUT_SECONDS: + if (sync_id in sync_states + and sync_states[sync_id].get('status') in _SYNC_TERMINAL_STATUSES): + break + time.sleep(2) + elapsed = int(time.time() - sync_poll_start) + sub_progress = progress_start + 1 + ((pl_idx + 1) / n_playlists) * progress_span + deps.update_progress( + automation_id, + progress=min(int(sub_progress), progress_end - 1), + phase=f'{sync_phase_label.rstrip(".")} — "{pl_name}" ({elapsed}s)', + ) + + ss = sync_states.get(sync_id, {}) + ss_result = ss.get('result', ss.get('progress', {})) + matched = ss_result.get('matched_tracks', 0) if isinstance(ss_result, dict) else 0 + total_synced += int(matched) if matched else 0 + deps.update_progress( + automation_id, + log_line=f'Synced "{pl_name}": {matched} tracks matched', + log_type='success', + ) + + elif sync_status == 'skipped': + total_skipped += 1 + reason = sync_result.get('reason', 'unchanged') + deps.update_progress( + automation_id, + log_line=f'Skipped "{pl_name}": {reason}', + log_type='skip', + ) + elif sync_status == 'error': + sync_errors += 1 + deps.update_progress( + automation_id, + log_line=f'Sync error "{pl_name}": {sync_result.get("reason", "unknown")}', + log_type='error', + ) + + deps.update_progress( + automation_id, + progress=progress_end, + phase=f'{sync_phase_label.rstrip(".")} complete', + log_line=f'Sync done: {total_synced} matched, {total_skipped} skipped, {sync_errors} errors', + log_type='success' if sync_errors == 0 else 'warning', + ) + + wishlist_queued = run_wishlist_phase( + deps, automation_id, + skip=skip_wishlist, + progress_pct=progress_end + 1, + wishlist_phase_label=wishlist_phase_label, + wishlist_phase_start_log=wishlist_phase_start_log, + ) + + return { + 'synced': total_synced, + 'skipped': total_skipped, + 'errors': sync_errors, + 'wishlist_queued': wishlist_queued, + } + + +def run_wishlist_phase( + deps: AutomationDeps, + automation_id: Optional[str], + *, + skip: bool, + progress_pct: int, + wishlist_phase_label: str = 'Phase: Processing wishlist...', + wishlist_phase_start_log: str = 'Wishlist', +) -> int: + """Trigger the wishlist processor unless skipped or already running. + + Returns 1 when the processor was triggered, 0 otherwise. Errors are + logged but never raised — wishlist failure should not abort the + pipeline.""" + if skip: + deps.update_progress( + automation_id, + progress=progress_pct, + log_line=f'{wishlist_phase_start_log}: skipped (disabled)', + log_type='skip', + ) + return 0 + + deps.update_progress( + automation_id, + progress=progress_pct, + phase=wishlist_phase_label, + log_line=wishlist_phase_start_log, + log_type='info', + ) + + try: + if not deps.is_wishlist_actually_processing(): + deps.process_wishlist_automatically(automation_id=None) + deps.update_progress( + automation_id, + log_line='Wishlist processing triggered', + log_type='success', + ) + return 1 + deps.update_progress( + automation_id, + log_line='Wishlist already running — skipped', + log_type='skip', + ) + return 0 + except Exception as e: # noqa: BLE001 — wishlist failure must never abort pipeline + deps.update_progress( + automation_id, + log_line=f'Wishlist error: {e}', + log_type='warning', + ) + return 0 + + +__all__ = ['run_sync_and_wishlist', 'run_wishlist_phase'] diff --git a/core/automation/handlers/database_update.py b/core/automation/handlers/database_update.py new file mode 100644 index 00000000..a9d51036 --- /dev/null +++ b/core/automation/handlers/database_update.py @@ -0,0 +1,136 @@ +"""Automation handlers: ``start_database_update`` and +``deep_scan_library`` actions. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_start_database_update`` and ``_auto_deep_scan_library`` +closures). Both share the same ``db_update_state`` / executor / lock +infrastructure -- the only difference is which task they submit +(``run_db_update_task`` vs ``run_deep_scan_task``). + +Pattern: pre-set state to running, submit task to executor, then +poll the state dict until it transitions away from ``running``. +Stall-detection emits a warning every 10 minutes when progress +hasn't budged. 2-hour outer timeout caps the worst case. +""" + +from __future__ import annotations + +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +_TIMEOUT_SECONDS = 7200 # 2 hours — covers the worst large-library case +_STALL_WARNING_SECONDS = 600 # 10 minutes without progress = stall +_POLL_INTERVAL_SECONDS = 3 +_INITIAL_DELAY_SECONDS = 1 + + +def auto_start_database_update(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run a full or incremental DB update via ``run_db_update_task``.""" + return _run_with_progress( + config, deps, + task=deps.run_db_update_task, + task_args=(config.get('full_refresh', False), deps.config_manager.get_active_media_server()), + initial_phase='Initializing...', + stall_label='Database update', + finished_extras=lambda: {'full_refresh': str(config.get('full_refresh', False))}, + timeout_label='Database update timed out after 2 hours', + ) + + +def auto_deep_scan_library(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run a deep library scan via ``run_deep_scan_task``.""" + return _run_with_progress( + config, deps, + task=deps.run_deep_scan_task, + task_args=(deps.config_manager.get_active_media_server(),), + initial_phase='Deep scan: Initializing...', + stall_label='Deep scan', + finished_extras=lambda: {}, + timeout_label='Deep scan timed out after 2 hours', + ) + + +def _run_with_progress( + config: Dict[str, Any], + deps: AutomationDeps, + *, + task, + task_args: tuple, + initial_phase: str, + stall_label: str, + finished_extras, + timeout_label: str, +) -> Dict[str, Any]: + """Shared poll-and-wait body for both DB-update handlers.""" + automation_id = config.get('_automation_id') + state = deps.get_db_update_state() + if state.get('status') == 'running': + return {'status': 'skipped', 'reason': 'Database update already running'} + deps.state.db_update_automation_id = automation_id + # Sync legacy module global so the DB-update progress callbacks + # (still living in web_server.py) emit against this automation. + deps.set_db_update_automation_id(automation_id) + + with deps.db_update_lock: + state.update({ + 'status': 'running', 'phase': initial_phase, + 'progress': 0, 'current_item': '', 'processed': 0, 'total': 0, + 'error_message': '', + }) + deps.db_update_executor.submit(task, *task_args) + + # Monitor progress (callbacks handle card updates, we just block until done). + time.sleep(_INITIAL_DELAY_SECONDS) + poll_start = time.time() + last_progress_time = time.time() + last_progress_val = 0 + while time.time() - poll_start < _TIMEOUT_SECONDS: + time.sleep(_POLL_INTERVAL_SECONDS) + with deps.db_update_lock: + current_status = state.get('status', 'idle') + current_progress = state.get('progress', 0) + if current_status != 'running': + break + # Stall detection — if no progress change in 10 minutes, warn. + if current_progress != last_progress_val: + last_progress_val = current_progress + last_progress_time = time.time() + elif time.time() - last_progress_time > _STALL_WARNING_SECONDS: + deps.update_progress( + automation_id, + log_line=f'{stall_label} appears stalled — waiting...', + log_type='warning', + ) + last_progress_time = time.time() # Reset so warning repeats every 10 min. + else: + # 2-hour timeout reached. + deps.update_progress( + automation_id, status='error', + phase='Timed out', log_line=timeout_label, log_type='error', + ) + return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} + + # Finished/error callback already updated the card — return matching status. + with deps.db_update_lock: + final_status = state.get('status', 'unknown') + if final_status == 'error': + return { + 'status': 'error', + 'reason': state.get('error_message', 'Unknown error'), + '_manages_own_progress': True, + } + with deps.db_update_lock: + stats = { + 'status': 'completed', '_manages_own_progress': True, + 'artists': state.get('total', 0), + 'albums': state.get('total_albums', 0), + 'tracks': state.get('total_tracks', 0), + 'removed_artists': state.get('removed_artists', 0), + 'removed_albums': state.get('removed_albums', 0), + 'removed_tracks': state.get('removed_tracks', 0), + } + stats.update(finished_extras()) + return stats diff --git a/core/automation/handlers/discover_playlist.py b/core/automation/handlers/discover_playlist.py new file mode 100644 index 00000000..90edd2a2 --- /dev/null +++ b/core/automation/handlers/discover_playlist.py @@ -0,0 +1,48 @@ +"""Automation handler: ``discover_playlist`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_discover_playlist`` closure). Kicks off background discovery +of official Spotify / iTunes metadata for mirrored playlist tracks. +The worker runs in a daemon thread and emits its own progress; this +handler returns immediately after launching it (``_manages_own_progress``). +""" + +from __future__ import annotations + +import threading +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_discover_playlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Discover official Spotify/iTunes metadata for mirrored + playlist tracks. Runs the worker in a background thread.""" + db = deps.get_database() + playlist_id = config.get('playlist_id') + discover_all = config.get('all', False) + + if discover_all: + playlists = db.get_mirrored_playlists() + elif playlist_id: + p = db.get_mirrored_playlist(int(playlist_id)) + playlists = [p] if p else [] + else: + return {'status': 'error', 'reason': 'No playlist specified'} + + if not playlists: + return {'status': 'error', 'reason': 'No playlists found'} + + threading.Thread( + target=deps.run_playlist_discovery_worker, + args=(playlists, config.get('_automation_id')), + daemon=True, + name='auto-discover-playlist', + ).start() + names = ', '.join(p['name'] for p in playlists[:3]) + return { + 'status': 'started', + 'playlist_count': str(len(playlists)), + 'playlists': names, + '_manages_own_progress': True, + } diff --git a/core/automation/handlers/download_cleanup.py b/core/automation/handlers/download_cleanup.py new file mode 100644 index 00000000..4aba711a --- /dev/null +++ b/core/automation/handlers/download_cleanup.py @@ -0,0 +1,267 @@ +"""Automation handlers: download-queue cleanup actions. + +Lifted from ``web_server._register_automation_handlers``: +- ``clean_search_history`` → :func:`auto_clean_search_history` +- ``clean_completed_downloads`` → :func:`auto_clean_completed_downloads` +- ``full_cleanup`` → :func:`auto_full_cleanup` + +All three share the download-orchestrator + tasks_lock / +download_batches / download_tasks accessors. ``full_cleanup`` is a +multi-step orchestration that pulls in quarantine purge + staging +sweep on top of the queue cleanup -- kept as one big handler since +its phases share state-detection logic. +""" + +from __future__ import annotations + +import os +import shutil as _shutil +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +# ─── clean_search_history ──────────────────────────────────────────── + + +def auto_clean_search_history(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Remove old searches from Soulseek when configured.""" + automation_id = config.get('_automation_id') + # Skip if soulseek is not the active download source or in hybrid order. + dl_mode = deps.config_manager.get('download_source.mode', 'hybrid') + hybrid_order = deps.config_manager.get( + 'download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'], + ) + soulseek_active = ( + dl_mode == 'soulseek' + or (dl_mode == 'hybrid' and 'soulseek' in hybrid_order) + ) + # Reach the underlying SoulseekClient via the orchestrator's + # generic accessor. + slskd = deps.download_orchestrator.client('soulseek') if deps.download_orchestrator else None + if not soulseek_active or not slskd or not slskd.base_url: + deps.update_progress(automation_id, log_line='Soulseek not active — skipped', log_type='skip') + return {'status': 'skipped'} + if not deps.config_manager.get('soulseek.auto_clear_searches', True): + deps.update_progress( + automation_id, log_line='Auto-clear disabled in settings', log_type='skip', + ) + return {'status': 'skipped'} + try: + success = deps.run_async(deps.download_orchestrator.maintain_search_history_with_buffer( + keep_searches=50, trigger_threshold=200, + )) + if success: + deps.update_progress( + automation_id, + log_line='Search history maintenance completed', + log_type='success', + ) + return {'status': 'completed'} + else: + deps.update_progress(automation_id, log_line='No cleanup needed', log_type='skip') + return {'status': 'completed'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise + return {'status': 'error', 'error': str(e)} + + +# ─── clean_completed_downloads ─────────────────────────────────────── + + +def auto_clean_completed_downloads(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Clear completed downloads + sweep empty download directories. + Skips when active batches or post-processing is in flight.""" + automation_id = config.get('_automation_id') + try: + has_active_batches = False + has_post_processing = False + with deps.tasks_lock: + batches = deps.get_download_batches() + for batch_data in batches.values(): + if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: + has_active_batches = True + break + if not has_active_batches: + tasks = deps.get_download_tasks() + for task_data in tasks.values(): + if task_data.get('status') == 'post_processing': + has_post_processing = True + break + + if has_active_batches: + deps.update_progress( + automation_id, log_line='Skipped — downloads active', log_type='skip', + ) + return {'status': 'completed'} + + deps.run_async(deps.download_orchestrator.clear_all_completed_downloads()) + if not has_post_processing: + deps.sweep_empty_download_directories() + deps.update_progress( + automation_id, log_line='Download cleanup completed', log_type='success', + ) + return {'status': 'completed'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise + return {'status': 'error', 'reason': str(e)} + + +# ─── full_cleanup ──────────────────────────────────────────────────── + + +def auto_full_cleanup(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run all cleanup tasks: quarantine purge → download queue clear + → empty-dir sweep → staging sweep → search history.""" + automation_id = config.get('_automation_id') + steps = [] + + # --- 1. Clear quarantine --- + deps.update_progress(automation_id, phase='Clearing quarantine...', progress=0) + quarantine_path = os.path.join( + deps.docker_resolve_path(deps.config_manager.get('soulseek.download_path', './downloads')), + 'ss_quarantine', + ) + q_removed = 0 + if os.path.exists(quarantine_path): + for f in os.listdir(quarantine_path): + fp = os.path.join(quarantine_path, f) + try: + if os.path.isfile(fp): + os.remove(fp) + q_removed += 1 + elif os.path.isdir(fp): + _shutil.rmtree(fp) + q_removed += 1 + except Exception as e: # noqa: BLE001 — best-effort purge + deps.logger.debug("quarantine entry purge failed: %s", e) + steps.append(f'Quarantine: removed {q_removed} items') + deps.update_progress( + automation_id, + log_line=f'Quarantine: removed {q_removed} items', + log_type='success' if q_removed else 'info', + ) + + # --- 2. Clear completed/errored/cancelled downloads from Soulseek queue --- + deps.update_progress(automation_id, phase='Clearing download queue...', progress=20) + has_active_batches = False + has_post_processing = False + with deps.tasks_lock: + batches = deps.get_download_batches() + for batch_data in batches.values(): + if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: + has_active_batches = True + break + if not has_active_batches: + tasks = deps.get_download_tasks() + for task_data in tasks.values(): + if task_data.get('status') == 'post_processing': + has_post_processing = True + break + if has_active_batches: + steps.append('Download queue: skipped (active batches)') + deps.update_progress( + automation_id, + log_line='Download queue: skipped (active batches)', + log_type='skip', + ) + else: + try: + deps.run_async(deps.download_orchestrator.clear_all_completed_downloads()) + steps.append('Download queue: cleared') + deps.update_progress( + automation_id, log_line='Download queue: cleared', log_type='success', + ) + except Exception as e: # noqa: BLE001 — per-step best-effort + steps.append(f'Download queue: error ({e})') + deps.update_progress( + automation_id, + log_line=f'Download queue: error ({e})', + log_type='error', + ) + + # --- 3. Sweep empty download directories --- + deps.update_progress(automation_id, phase='Sweeping empty directories...', progress=40) + if has_active_batches or has_post_processing: + reason = 'active batches' if has_active_batches else 'post-processing active' + steps.append(f'Empty directories: skipped ({reason})') + deps.update_progress( + automation_id, + log_line=f'Empty directories: skipped ({reason})', + log_type='skip', + ) + else: + dirs_removed = deps.sweep_empty_download_directories() + steps.append(f'Empty directories: removed {dirs_removed}') + deps.update_progress( + automation_id, + log_line=f'Empty directories: removed {dirs_removed}', + log_type='success' if dirs_removed else 'info', + ) + + # --- 4. Sweep empty staging directories --- + deps.update_progress(automation_id, phase='Sweeping import folder...', progress=60) + staging_path = deps.get_staging_path() + s_removed = 0 + if os.path.isdir(staging_path): + for dirpath, _dirnames, _filenames in os.walk(staging_path, topdown=False): + if os.path.normpath(dirpath) == os.path.normpath(staging_path): + continue + try: + entries = os.listdir(dirpath) + except OSError: + continue + visible = [e for e in entries if not e.startswith('.')] + if not visible: + for hidden in entries: + try: + os.remove(os.path.join(dirpath, hidden)) + except Exception as e: # noqa: BLE001 — best-effort + deps.logger.debug("hidden file cleanup failed: %s", e) + try: + os.rmdir(dirpath) + s_removed += 1 + except OSError: + pass + steps.append(f'Staging: removed {s_removed} empty directories') + deps.update_progress( + automation_id, + log_line=f'Staging: removed {s_removed} empty directories', + log_type='success' if s_removed else 'info', + ) + + # --- 5. Clean search history (if enabled) --- + deps.update_progress(automation_id, phase='Cleaning search history...', progress=80) + try: + if not deps.config_manager.get('soulseek.auto_clear_searches', True): + steps.append('Search cleanup: disabled in settings') + deps.update_progress( + automation_id, log_line='Search cleanup: disabled in settings', log_type='skip', + ) + else: + deps.run_async(deps.download_orchestrator.maintain_search_history_with_buffer( + keep_searches=50, trigger_threshold=200, + )) + steps.append('Search history: cleaned') + deps.update_progress( + automation_id, log_line='Search history: cleaned', log_type='success', + ) + except Exception as e: # noqa: BLE001 — per-step best-effort + steps.append(f'Search history: error ({e})') + deps.update_progress( + automation_id, log_line=f'Search history: error ({e})', log_type='error', + ) + + total_removed = q_removed + s_removed + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', + log_line=f'Full cleanup complete — {total_removed} items removed', + log_type='success', + ) + return { + 'status': 'completed', + 'quarantine_removed': str(q_removed), + 'staging_removed': str(s_removed), + 'total_removed': str(total_removed), + 'steps': steps, + '_manages_own_progress': True, + } diff --git a/core/automation/handlers/duplicate_cleaner.py b/core/automation/handlers/duplicate_cleaner.py new file mode 100644 index 00000000..a0531622 --- /dev/null +++ b/core/automation/handlers/duplicate_cleaner.py @@ -0,0 +1,87 @@ +"""Automation handler: ``run_duplicate_cleaner`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_run_duplicate_cleaner`` closure). Submits the duplicate +cleaner to its executor, then polls the shared state dict until +the worker transitions away from ``running``. +""" + +from __future__ import annotations + +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +_TIMEOUT_SECONDS = 7200 # 2 hours +_POLL_INTERVAL_SECONDS = 3 +_INITIAL_DELAY_SECONDS = 1 + + +def auto_run_duplicate_cleaner(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Kick off the duplicate cleaner and report final stats.""" + automation_id = config.get('_automation_id') + state = deps.get_duplicate_cleaner_state() + if state.get('status') == 'running': + return {'status': 'skipped', 'reason': 'Duplicate cleaner already running'} + + # Pre-set status before submit so the polling loop doesn't see a + # stale 'finished' from a previous run. + with deps.duplicate_cleaner_lock: + state['status'] = 'running' + deps.duplicate_cleaner_executor.submit(deps.run_duplicate_cleaner) + deps.update_progress(automation_id, log_line='Duplicate cleaner started', log_type='info') + + # Monitor progress (max 2 hours). + time.sleep(_INITIAL_DELAY_SECONDS) + poll_start = time.time() + while time.time() - poll_start < _TIMEOUT_SECONDS: + time.sleep(_POLL_INTERVAL_SECONDS) + current_status = state.get('status', 'idle') + if current_status not in ('running',): + break + deps.update_progress( + automation_id, + phase=state.get('phase', 'Scanning...'), + progress=state.get('progress', 0), + processed=state.get('files_scanned', 0), + total=state.get('total_files', 0), + ) + else: + # 2-hour timeout reached. + deps.update_progress( + automation_id, status='error', + phase='Timed out', + log_line='Duplicate cleaner timed out after 2 hours', + log_type='error', + ) + return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} + + # Check actual exit status (could be 'finished' or 'error'). + final_status = state.get('status', 'idle') + if final_status == 'error': + err = state.get('error_message', 'Unknown error') + deps.update_progress( + automation_id, status='error', progress=100, + phase='Error', log_line=err, log_type='error', + ) + return {'status': 'error', 'reason': err, '_manages_own_progress': True} + + dupes = state.get('duplicates_found', 0) + removed = state.get('deleted', 0) + space_freed = state.get('space_freed', 0) + scanned = state.get('files_scanned', 0) + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', + log_line=f'Found {dupes} duplicates, removed {removed} files', + log_type='success', + ) + return { + 'status': 'completed', '_manages_own_progress': True, + 'files_scanned': scanned, + 'duplicates_found': dupes, + 'files_deleted': removed, + 'space_freed_mb': round(space_freed / (1024 * 1024), 1), + } diff --git a/core/automation/handlers/maintenance.py b/core/automation/handlers/maintenance.py new file mode 100644 index 00000000..dacd13c8 --- /dev/null +++ b/core/automation/handlers/maintenance.py @@ -0,0 +1,213 @@ +"""Automation handlers: short maintenance actions. + +Lifted from ``web_server._register_automation_handlers``: +- ``clear_quarantine`` → :func:`auto_clear_quarantine` +- ``cleanup_wishlist`` → :func:`auto_cleanup_wishlist` +- ``update_discovery_pool`` → :func:`auto_update_discovery_pool` +- ``backup_database`` → :func:`auto_backup_database` +- ``refresh_beatport_cache`` → :func:`auto_refresh_beatport_cache` + +Each is a thin wrapper around an existing service / helper. Grouped +in one module because every body is short and they share no state +between them — splitting into per-handler files would just add +import noise. +""" + +from __future__ import annotations + +import glob as _glob +import os +import shutil as _shutil +import sqlite3 +import time +from datetime import datetime +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +# ─── clear_quarantine ──────────────────────────────────────────────── + + +def auto_clear_quarantine(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Purge every file/folder under the configured ss_quarantine path.""" + automation_id = config.get('_automation_id') + quarantine_path = os.path.join( + deps.docker_resolve_path(deps.config_manager.get('soulseek.download_path', './downloads')), + 'ss_quarantine', + ) + if not os.path.exists(quarantine_path): + deps.update_progress(automation_id, log_line='No quarantine folder found', log_type='info') + return {'status': 'completed', 'removed': '0'} + removed = 0 + for f in os.listdir(quarantine_path): + fp = os.path.join(quarantine_path, f) + try: + if os.path.isfile(fp): + os.remove(fp) + removed += 1 + elif os.path.isdir(fp): + _shutil.rmtree(fp) + removed += 1 + except Exception as e: # noqa: BLE001 — best-effort purge + deps.logger.debug("quarantine entry purge failed: %s", e) + deps.update_progress( + automation_id, + log_line=f'Removed {removed} quarantined items', + log_type='success' if removed > 0 else 'info', + ) + return {'status': 'completed', 'removed': str(removed)} + + +# ─── cleanup_wishlist ──────────────────────────────────────────────── + + +def auto_cleanup_wishlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Drop duplicate entries from the wishlist for the active profile.""" + automation_id = config.get('_automation_id') + db = deps.get_database() + removed = db.remove_wishlist_duplicates(deps.get_current_profile_id()) + deps.update_progress( + automation_id, + log_line=f'Removed {removed or 0} duplicate wishlist entries', + log_type='success' if removed else 'info', + ) + return {'status': 'completed', 'removed': str(removed or 0)} + + +# ─── update_discovery_pool ─────────────────────────────────────────── + + +def auto_update_discovery_pool(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run an incremental refresh of the discovery pool via the + watchlist scanner.""" + automation_id = config.get('_automation_id') + try: + scanner = deps.get_watchlist_scanner(deps.spotify_client) + deps.update_progress(automation_id, log_line='Updating discovery pool...', log_type='info') + scanner.update_discovery_pool_incremental(deps.get_current_profile_id()) + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', log_line='Discovery pool updated', log_type='success', + ) + return {'status': 'completed', '_manages_own_progress': True} + except Exception as e: # noqa: BLE001 — automation handlers must never raise + deps.update_progress( + automation_id, status='error', + phase='Error', log_line=str(e), log_type='error', + ) + return {'status': 'error', 'reason': str(e), '_manages_own_progress': True} + + +# ─── backup_database ───────────────────────────────────────────────── + + +_MAX_BACKUPS = 5 + + +def auto_backup_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Create a hot SQLite backup, then prune old backups so only the + newest ``_MAX_BACKUPS`` remain.""" + automation_id = config.get('_automation_id') + db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') + if not os.path.exists(db_path): + return {'status': 'error', 'reason': 'Database file not found'} + + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + backup_path = f"{db_path}.backup_{timestamp}" + # Use SQLite backup API for a safe hot-copy of an active database. + src = sqlite3.connect(db_path) + dst = sqlite3.connect(backup_path) + src.backup(dst) + dst.close() + src.close() + size_mb = round(os.path.getsize(backup_path) / (1024 * 1024), 1) + + # Rolling cleanup — keep only the newest N backups. + existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime) + while len(existing) > _MAX_BACKUPS: + try: + os.remove(existing.pop(0)) + except Exception as e: # noqa: BLE001 — best-effort cleanup + deps.logger.debug("rolling backup cleanup failed: %s", e) + deps.update_progress( + automation_id, + log_line=f'Backup created: {size_mb}MB ({os.path.basename(backup_path)})', + log_type='success', + ) + return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)} + + +# ─── refresh_beatport_cache ────────────────────────────────────────── + + +_BEATPORT_SECTIONS = ( + ('hero_tracks', '/api/beatport/hero-tracks', 'Hero Tracks'), + ('new_releases', '/api/beatport/new-releases', 'New Releases'), + ('featured_charts', '/api/beatport/featured-charts', 'Featured Charts'), + ('dj_charts', '/api/beatport/dj-charts', 'DJ Charts'), + ('top_10_lists', '/api/beatport/homepage/top-10-lists', 'Top 10 Lists'), + ('top_10_releases', '/api/beatport/homepage/top-10-releases-cards', 'Top 10 Releases'), + ('hype_picks', '/api/beatport/hype-picks', 'Hype Picks'), +) + + +def auto_refresh_beatport_cache(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Refresh Beatport homepage cache by calling each endpoint internally + via Flask's ``test_client``. Invalidates the homepage cache first + so endpoints re-scrape rather than returning stale data.""" + automation_id = config.get('_automation_id') + cache = deps.get_beatport_data_cache() + # Invalidate all homepage cache timestamps so endpoints re-scrape. + with cache['cache_lock']: + for key in cache['homepage']: + cache['homepage'][key]['timestamp'] = 0 + cache['homepage'][key]['data'] = None + + refreshed = 0 + errors = [] + app = deps.get_app() + with app.test_client() as client: + for idx, (_, endpoint, label) in enumerate(_BEATPORT_SECTIONS): + deps.update_progress( + automation_id, + progress=(idx / len(_BEATPORT_SECTIONS)) * 100, + phase=f'Scraping: {label}', + current_item=label, + ) + try: + resp = client.get(endpoint) + if resp.status_code == 200: + refreshed += 1 + deps.update_progress( + automation_id, log_line=f'{label}: cached', log_type='success', + ) + else: + errors.append(label) + deps.update_progress( + automation_id, + log_line=f'{label}: HTTP {resp.status_code}', + log_type='error', + ) + except Exception as e: # noqa: BLE001 — per-section best-effort + errors.append(label) + deps.update_progress( + automation_id, + log_line=f'{label}: {str(e)}', + log_type='error', + ) + if idx < len(_BEATPORT_SECTIONS) - 1: + time.sleep(2) + + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', + log_line=f'Refreshed {refreshed}/{len(_BEATPORT_SECTIONS)} sections', + log_type='success', + ) + return { + 'status': 'completed', + 'refreshed': str(refreshed), + 'errors': str(len(errors)), + '_manages_own_progress': True, + } diff --git a/core/automation/handlers/personalized_pipeline.py b/core/automation/handlers/personalized_pipeline.py new file mode 100644 index 00000000..aa2f9190 --- /dev/null +++ b/core/automation/handlers/personalized_pipeline.py @@ -0,0 +1,356 @@ +"""Personalized Playlist Pipeline automation handler. + +Sibling to ``auto_playlist_pipeline`` (mirrored). Where the mirrored +pipeline runs REFRESH external sources → DISCOVER metadata → SYNC → +WISHLIST, the personalized pipeline is simpler: + + SNAPSHOT → SYNC → WISHLIST + +SNAPSHOT reads the persisted track list from +``PersonalizedPlaylistManager``. When ``refresh_first=True`` (config), +each playlist is refreshed BEFORE syncing — useful when the user +wants the cron to capture a fresh-each-run view (e.g. "give me a new +Hidden Gems set every night"). Default is to sync the existing +snapshot, on the assumption the user / a separate cron has already +refreshed when they wanted to. + +Config schema: + { + 'kinds': [ + {'kind': 'hidden_gems'}, + {'kind': 'time_machine', 'variant': '1980s'}, + {'kind': 'seasonal_mix', 'variant': 'halloween'}, + ... + ], + 'refresh_first': bool, # default false + 'skip_wishlist': bool, # default false + } + +Each kind dict has at minimum ``kind``; ``variant`` is required for +kinds that need it (time_machine, genre_playlist, daily_mix, +seasonal_mix). Singleton kinds (hidden_gems, discovery_shuffle, +popular_picks, fresh_tape, archives) ignore variant. + +Pipeline-running flag (``deps.state.pipeline_running``) is shared +with the mirrored pipeline so the two can't overlap. (One sync +queue, one wishlist worker — overlapping triggers would step on +each other.)""" + +from __future__ import annotations + +import json +import threading +import time +from typing import Any, Dict, List, Optional + +from core.automation.deps import AutomationDeps +from core.automation.handlers._pipeline_shared import run_sync_and_wishlist + + +# Sync state key prefix so personalized syncs don't collide with +# mirrored ones (`auto_mirror_`). +_SYNC_ID_PREFIX = 'auto_personalized' + + +def auto_personalized_pipeline(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run SNAPSHOT → SYNC → WISHLIST for selected personalized playlists.""" + deps.state.set_pipeline_running(True) + automation_id = config.get('_automation_id') + pipeline_start = time.time() + + try: + kinds_config = config.get('kinds') or [] + if not isinstance(kinds_config, list) or not kinds_config: + deps.state.set_pipeline_running(False) + return { + 'status': 'error', + 'error': 'No personalized playlist kinds selected', + } + + refresh_first = bool(config.get('refresh_first', False)) + skip_wishlist = bool(config.get('skip_wishlist', False)) + + manager = deps.build_personalized_manager() + + deps.update_progress( + automation_id, + progress=2, + phase=f'Personalized pipeline: {len(kinds_config)} playlist(s)', + log_line=f'Starting pipeline for {len(kinds_config)} playlist(s)', + log_type='info', + ) + + # ── PHASE 1: SNAPSHOT (optionally refresh) ────────────────── + deps.update_progress( + automation_id, + progress=3, + phase='Phase 1/2: Loading snapshots...' if not refresh_first + else 'Phase 1/2: Refreshing snapshots...', + log_line='Phase 1: Snapshot' + (' (with refresh)' if refresh_first else ''), + log_type='info', + ) + + profile_id = deps.get_current_profile_id() + playload_payloads = _build_payloads_for_kinds( + deps, manager, kinds_config, profile_id, + automation_id=automation_id, + refresh_first=refresh_first, + ) + + if not playload_payloads: + deps.state.set_pipeline_running(False) + deps.update_progress( + automation_id, + status='finished', progress=100, + phase='No playlists to sync', + log_line='No personalized playlists had tracks to sync', + log_type='warning', + ) + return { + 'status': 'completed', + '_manages_own_progress': True, + 'playlists_synced': '0', + 'tracks_synced': '0', + 'duration_seconds': str(int(time.time() - pipeline_start)), + } + + deps.update_progress( + automation_id, + progress=50, + phase='Phase 1/2: Snapshot complete', + log_line=f'Phase 1 done: {len(playload_payloads)} playlist(s) ready to sync', + log_type='success', + ) + + # ── PHASE 2: SYNC + WISHLIST (shared helper) ──────────────── + sync_summary = run_sync_and_wishlist( + deps, + automation_id, + playload_payloads, + sync_one_fn=lambda pl: _sync_personalized_playlist(deps, pl), + sync_id_for_fn=lambda pl: pl['sync_id'], + skip_wishlist=skip_wishlist, + progress_start=51, + progress_end=90, + sync_phase_label='Phase 2/2: Syncing to server...', + sync_phase_start_log='Phase 2: Sync', + wishlist_phase_label='Phase 2/2: Processing wishlist...', + wishlist_phase_start_log='Wishlist', + ) + + # ── COMPLETE ──────────────────────────────────────────────── + duration = int(time.time() - pipeline_start) + deps.update_progress( + automation_id, + status='finished', progress=100, + phase='Pipeline complete', + log_line=f'Personalized pipeline finished in {duration // 60}m {duration % 60}s', + log_type='success', + ) + + deps.state.set_pipeline_running(False) + return { + 'status': 'completed', + '_manages_own_progress': True, + 'playlists_synced': str(len(playload_payloads)), + 'tracks_synced': str(sync_summary['synced']), + 'sync_skipped': str(sync_summary['skipped']), + 'wishlist_queued': str(sync_summary['wishlist_queued']), + 'duration_seconds': str(duration), + } + + except Exception as e: # noqa: BLE001 — automation handlers must never raise into engine + deps.state.set_pipeline_running(False) + deps.update_progress( + automation_id, + status='error', progress=100, + phase='Pipeline error', + log_line=f'Personalized pipeline failed: {e}', + log_type='error', + ) + return {'status': 'error', 'error': str(e), '_manages_own_progress': True} + + +def _build_payloads_for_kinds( + deps: AutomationDeps, + manager: Any, + kinds_config: List[Dict[str, Any]], + profile_id: int, + *, + automation_id: Optional[str], + refresh_first: bool, +) -> List[Dict[str, Any]]: + """Resolve each requested kind+variant into a sync-payload dict. + + Each payload has: ``{'name', 'kind', 'variant', 'tracks_json', + 'image_url', 'sync_id'}``. Playlists with no tracks (e.g. a + seasonal mix that hasn't been populated yet) are omitted from + the result so the sync loop doesn't waste time on empty pushes. + """ + payloads: List[Dict[str, Any]] = [] + for entry in kinds_config: + if not isinstance(entry, dict): + continue + kind = entry.get('kind') + variant = entry.get('variant') or '' + if not kind: + continue + + try: + # Refresh when ANY of: + # - explicit user flag (cron use case: regenerate each run) + # - snapshot marked stale by upstream data refresher + # - playlist was never generated yet (auto-created by + # ensure_playlist; track_count=0, last_generated_at=NULL). + # Without this branch, a first-run pipeline reads the + # empty snapshot and silently skips — user picks a kind, + # hits run, gets "No tracks to sync" with no clue why. + if refresh_first: + record = manager.refresh_playlist(kind, variant, profile_id) + else: + existing = manager.ensure_playlist(kind, variant, profile_id) + needs_first_gen = existing.last_generated_at is None + if existing.is_stale or needs_first_gen: + record = manager.refresh_playlist(kind, variant, profile_id) + else: + record = existing + except Exception as exc: # noqa: BLE001 — log + continue with next kind + deps.update_progress( + automation_id, + log_line=f'Skipping {kind}{("/" + variant) if variant else ""}: {exc}', + log_type='warning', + ) + continue + + tracks = manager.get_playlist_tracks(record.id) + if not tracks: + deps.update_progress( + automation_id, + log_line=f'No tracks in {record.name} — skipping sync', + log_type='skip', + ) + continue + + tracks_json = [_track_to_sync_shape(t) for t in tracks] + payloads.append({ + 'name': record.name, + 'kind': record.kind, + 'variant': record.variant, + 'tracks_json': tracks_json, + 'image_url': '', # personalized playlists don't have a cover image yet + 'sync_id': f'{_SYNC_ID_PREFIX}_{record.kind}_{record.variant or "_"}', + }) + return payloads + + +def _track_to_sync_shape(track: Any) -> Dict[str, Any]: + """Convert a personalized.types.Track into the dict shape + `_run_sync_task` expects. Mirrors what the mirrored pipeline + builds from extra_data.matched_data, preserving enriched metadata + from personalized snapshots when available.""" + primary_id = track.spotify_track_id or track.itunes_track_id or track.deezer_track_id or '' + rich_data = _coerce_track_data_json(getattr(track, 'track_data_json', None)) + + if not rich_data: + album = {'name': track.album_name or ''} + cover_url = getattr(track, 'album_cover_url', None) + if cover_url: + album['images'] = [{'url': cover_url}] + return { + 'name': track.track_name, + 'artists': [{'name': track.artist_name}], + 'album': album, + 'duration_ms': int(track.duration_ms or 0), + 'id': primary_id, + } + + payload = dict(rich_data) + cover_url = ( + getattr(track, 'album_cover_url', None) + or payload.get('album_cover_url') + or payload.get('image_url') + ) + payload['id'] = payload.get('id') or primary_id + payload['name'] = payload.get('name') or track.track_name + payload['artists'] = _normalize_artists(payload.get('artists'), track.artist_name) + payload['album'] = _normalize_album(payload.get('album'), track, cover_url=cover_url) + payload['duration_ms'] = int(payload.get('duration_ms') or track.duration_ms or 0) + if 'popularity' not in payload and getattr(track, 'popularity', None) is not None: + payload['popularity'] = int(track.popularity or 0) + + if cover_url and not payload.get('image_url'): + payload['image_url'] = cover_url + + return payload + + +def _coerce_track_data_json(value: Any) -> Dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, str) and value.strip(): + try: + loaded = json.loads(value) + except (TypeError, ValueError): + return {} + return loaded if isinstance(loaded, dict) else {} + return {} + + +def _normalize_artists(artists: Any, fallback_artist: str) -> List[Dict[str, Any]]: + if not artists: + return [{'name': fallback_artist or 'Unknown Artist'}] + if isinstance(artists, list): + normalized = [] + for artist in artists: + if isinstance(artist, dict): + normalized.append(artist if artist.get('name') else {'name': str(artist)}) + elif isinstance(artist, str): + normalized.append({'name': artist}) + else: + normalized.append({'name': str(artist)}) + return normalized or [{'name': fallback_artist or 'Unknown Artist'}] + if isinstance(artists, dict): + return [artists if artists.get('name') else {'name': str(artists)}] + return [{'name': str(artists)}] + + +def _normalize_album(album: Any, track: Any, cover_url: Optional[str] = None) -> Dict[str, Any]: + if isinstance(album, dict): + normalized = dict(album) + else: + normalized = {'name': str(album) if album else (track.album_name or '')} + + normalized['name'] = normalized.get('name') or track.album_name or '' + images = normalized.get('images') + if not isinstance(images, list): + images = [] + if cover_url and not images: + images = [{'url': cover_url}] + if images: + normalized['images'] = images + return normalized + + +def _sync_personalized_playlist(deps: AutomationDeps, payload: Dict[str, Any]) -> Dict[str, Any]: + """Launch a personalized playlist sync via _run_sync_task on a + daemon thread + return immediately with status='started'. + + Mirrors the mirrored ``auto_sync_playlist`` return contract so the + shared helper can poll on ``sync_states[sync_id]`` and aggregate + results identically.""" + sync_id = payload['sync_id'] + name = payload['name'] + tracks_json = payload['tracks_json'] + profile_id = deps.get_current_profile_id() + + threading.Thread( + target=deps.run_sync_task, + args=(sync_id, name, tracks_json, None, profile_id, payload.get('image_url', '')), + daemon=True, + name=f'auto-personalized-{sync_id}', + ).start() + return { + 'status': 'started', + 'playlist_name': name, + '_manages_own_progress': True, + } diff --git a/core/automation/handlers/playlist_pipeline.py b/core/automation/handlers/playlist_pipeline.py new file mode 100644 index 00000000..8fc98df2 --- /dev/null +++ b/core/automation/handlers/playlist_pipeline.py @@ -0,0 +1,227 @@ +"""Automation handler: ``playlist_pipeline`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_playlist_pipeline`` closure). Runs the full playlist +lifecycle in a single trigger: + + Phase 1: REFRESH -- pull fresh track lists from sources + Phase 2: DISCOVER -- look up official Spotify/iTunes metadata + Phase 3: SYNC -- push the result to the active media server + Phase 4: WISHLIST -- queue any missing tracks for download + +Each phase emits its own progress range so the trigger card shows +useful per-phase percentages instead of "loading...". Phase 4 is +optional via ``skip_wishlist`` config. + +Composition: this handler invokes ``auto_refresh_mirrored`` and +``auto_sync_playlist`` directly (passing ``_automation_id: None`` so +the sub-handlers don't hijack pipeline progress) instead of going +through the engine — keeps the four phases observable as one +trigger from the user's perspective. Pipeline-level guard +(``state.pipeline_running``) prevents overlapping runs. +""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps +from core.automation.handlers._pipeline_shared import run_sync_and_wishlist +from core.automation.handlers.refresh_mirrored import auto_refresh_mirrored +from core.automation.handlers.sync_playlist import auto_sync_playlist + + +# Per-playlist sync poll cap inside Phase 3. +# Discovery poll cap inside Phase 2. +_DISCOVERY_TIMEOUT_SECONDS = 3600 + + +def auto_playlist_pipeline(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run REFRESH → DISCOVER → SYNC → WISHLIST in sequence. + + Sets / clears ``deps.state.pipeline_running`` around the whole + run so the registration guard can short-circuit overlapping + triggers. + """ + deps.state.set_pipeline_running(True) + automation_id = config.get('_automation_id') + pipeline_start = time.time() + + try: + db = deps.get_database() + playlist_id = config.get('playlist_id') + process_all = config.get('all', False) + skip_wishlist = config.get('skip_wishlist', False) + + # Resolve playlists. + if process_all: + playlists = db.get_mirrored_playlists() + elif playlist_id: + p = db.get_mirrored_playlist(int(playlist_id)) + playlists = [p] if p else [] + else: + deps.state.set_pipeline_running(False) + return {'status': 'error', 'error': 'No playlist specified'} + + playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] + if not playlists: + deps.state.set_pipeline_running(False) + return {'status': 'error', 'error': 'No refreshable playlists found'} + + pl_names = ', '.join(p.get('name', '?') for p in playlists[:3]) + if len(playlists) > 3: + pl_names += f' (+{len(playlists) - 3} more)' + + deps.update_progress( + automation_id, + progress=2, + phase=f'Pipeline: {len(playlists)} playlist(s)', + log_line=f'Starting pipeline for: {pl_names}', + log_type='info', + ) + + # ── PHASE 1: REFRESH ────────────────────────────────────────── + deps.update_progress( + automation_id, + progress=3, + phase='Phase 1/4: Refreshing playlists...', + log_line='Phase 1: Refresh', + log_type='info', + ) + + refresh_config = dict(config) + refresh_config['_automation_id'] = None # Don't let sub-handler hijack pipeline progress. + refresh_result = auto_refresh_mirrored(refresh_config, deps) + refreshed = int(refresh_result.get('refreshed', 0)) + refresh_errors = int(refresh_result.get('errors', 0)) + + deps.update_progress( + automation_id, + progress=25, + phase='Phase 1/4: Refresh complete', + log_line=f'Phase 1 done: {refreshed} refreshed, {refresh_errors} errors', + log_type='success' if refresh_errors == 0 else 'warning', + ) + + # ── PHASE 2: DISCOVER ───────────────────────────────────────── + deps.update_progress( + automation_id, + progress=26, + phase='Phase 2/4: Discovering metadata...', + log_line='Phase 2: Discover', + log_type='info', + ) + + # Reload playlists (refresh may have updated them). + if process_all: + disc_playlists = db.get_mirrored_playlists() + else: + disc_playlists = [db.get_mirrored_playlist(int(playlist_id))] + disc_playlists = [p for p in disc_playlists if p] + + # Run discovery in a thread and wait for it. + disc_done = threading.Event() + + def _disc_wrapper(pls): + try: + # The worker updates automation_progress internally, + # but we pass None so it doesn't conflict with our + # pipeline progress. + deps.run_playlist_discovery_worker(pls, automation_id=None) + except Exception as e: + deps.logger.error(f"[Pipeline] Discovery error: {e}") + finally: + disc_done.set() + + threading.Thread( + target=_disc_wrapper, args=(disc_playlists,), + daemon=True, name='pipeline-discover', + ).start() + + # Poll for completion with progress updates. + poll_start = time.time() + while not disc_done.wait(timeout=3): + elapsed = int(time.time() - poll_start) + deps.update_progress( + automation_id, + progress=min(26 + elapsed // 4, 54), + phase=f'Phase 2/4: Discovering... ({elapsed}s)', + ) + if elapsed > _DISCOVERY_TIMEOUT_SECONDS: + deps.update_progress( + automation_id, + log_line='Discovery timed out after 1 hour', + log_type='warning', + ) + break + + deps.update_progress( + automation_id, + progress=55, + phase='Phase 2/4: Discovery complete', + log_line='Phase 2 done: discovery complete', + log_type='success', + ) + + # ── PHASE 3 + 4: SYNC + WISHLIST (delegated to shared helper) ── + # Each mirrored playlist payload only needs `id` + `name` for + # the helper; `auto_sync_playlist` reads the rest from the + # mirrored DB by id. + sync_summary = run_sync_and_wishlist( + deps, + automation_id, + [pl for pl in playlists if pl.get('id')], + sync_one_fn=lambda pl: auto_sync_playlist( + {'playlist_id': str(pl['id']), '_automation_id': None}, + deps, + ), + sync_id_for_fn=lambda pl: f"auto_mirror_{pl['id']}", + skip_wishlist=skip_wishlist, + progress_start=56, + progress_end=85, + sync_phase_label='Phase 3/4: Syncing to server...', + sync_phase_start_log='Phase 3: Sync', + wishlist_phase_label='Phase 4/4: Processing wishlist...', + wishlist_phase_start_log='Phase 4: Wishlist', + ) + total_synced = sync_summary['synced'] + total_skipped = sync_summary['skipped'] + sync_errors = sync_summary['errors'] + wishlist_queued = sync_summary['wishlist_queued'] + + # ── COMPLETE ────────────────────────────────────────────────── + duration = int(time.time() - pipeline_start) + deps.update_progress( + automation_id, + status='finished', + progress=100, + phase='Pipeline complete', + log_line=f'Pipeline finished in {duration // 60}m {duration % 60}s', + log_type='success', + ) + + deps.state.set_pipeline_running(False) + return { + 'status': 'completed', + '_manages_own_progress': True, + 'playlists_refreshed': str(refreshed), + 'tracks_discovered': 'completed', + 'tracks_synced': str(total_synced), + 'sync_skipped': str(total_skipped), + 'wishlist_queued': str(wishlist_queued), + 'duration_seconds': str(duration), + } + + except Exception as e: + deps.state.set_pipeline_running(False) + deps.update_progress( + automation_id, + status='error', + progress=100, + phase='Pipeline error', + log_line=f'Pipeline failed: {e}', + log_type='error', + ) + return {'status': 'error', 'error': str(e), '_manages_own_progress': True} diff --git a/core/automation/handlers/process_wishlist.py b/core/automation/handlers/process_wishlist.py new file mode 100644 index 00000000..27a08046 --- /dev/null +++ b/core/automation/handlers/process_wishlist.py @@ -0,0 +1,27 @@ +"""Automation handler: ``process_wishlist`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_process_wishlist`` closure). Wishlist processing is async — +the helper submits a batch to an executor and returns immediately; +per-track stats arrive later via batch-completion callbacks. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_process_wishlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Kick off the wishlist processor for an automation trigger. + + Returns immediately after submission; the wishlist worker emits + per-batch progress via its own callbacks. We only report + ``status: completed`` to mark the trigger fired successfully. + """ + try: + deps.process_wishlist_automatically(automation_id=config.get('_automation_id')) + return {'status': 'completed'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise into the engine + return {'status': 'error', 'error': str(e)} diff --git a/core/automation/handlers/progress_callbacks.py b/core/automation/handlers/progress_callbacks.py new file mode 100644 index 00000000..a097864b --- /dev/null +++ b/core/automation/handlers/progress_callbacks.py @@ -0,0 +1,89 @@ +"""Progress + history callbacks the automation engine invokes around +each handler run. + +Lifted from the closures at the bottom of +``web_server._register_automation_handlers``: +- ``_progress_init`` → :func:`progress_init` +- ``_progress_finish`` → :func:`progress_finish` +- ``_record_automation_history`` → :func:`record_history` +- ``_on_library_scan_completed`` → :func:`on_library_scan_completed` + +The engine accepts four callables via +``register_progress_callbacks(init, finish, update, history)``; +``registration.register_all`` wires these here. The +``library_scan_completed`` callback is registered separately on the +``web_scan_manager`` (when one is available) -- see +``register_library_scan_completed_emitter``. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def progress_init(aid: Any, name: str, action_type: str, deps: AutomationDeps) -> None: + """Initialize per-automation progress state when the engine starts + a handler. Thin wrapper so the engine receives a closure that + delegates into the live progress tracker.""" + deps.init_automation_progress(aid, name, action_type) + + +def progress_finish(aid: Any, result: Dict[str, Any], deps: AutomationDeps) -> None: + """Emit the final progress update when a handler returns. + + Skipped for handlers that manage their own progress lifecycle + (they call ``update_progress(status='finished')`` themselves and + set ``_manages_own_progress: True`` in the returned dict). + Otherwise translates the handler's status into a finished/error + progress emit with a status-appropriate phase + log line. + """ + if result.get('_manages_own_progress'): + return + result_status = result.get('status', '') + status = 'error' if result_status == 'error' else 'finished' + msg = result.get('error', result.get('reason', result_status or 'done')) + deps.update_progress( + aid, + status=status, + progress=100, + phase='Error' if status == 'error' else 'Complete', + log_line=msg, + log_type='error' if status == 'error' else 'success', + ) + + +def record_history(aid: Any, result: Dict[str, Any], deps: AutomationDeps) -> None: + """Capture progress state into run history before the engine's + cleanup pass clears it. Thin wrapper so the engine sees a stable + callable.""" + deps.record_progress_history(aid, result, deps.get_database()) + + +def on_library_scan_completed(deps: AutomationDeps) -> None: + """Emit the ``library_scan_completed`` automation event with the + active media-server type. Replaces the hard-coded + ``scan_completion_callback → trigger_automatic_database_update`` + chain so any automation can listen for scan completion as a + trigger.""" + if not deps.engine: + return + server_type = ( + getattr(deps.web_scan_manager, '_current_server_type', None) + or 'unknown' + ) + deps.engine.emit('library_scan_completed', { + 'server_type': server_type, + }) + + +def register_library_scan_completed_emitter(deps: AutomationDeps) -> None: + """Wire :func:`on_library_scan_completed` to the + ``web_scan_manager``'s scan-completion callback list. No-op when + no scan manager is configured (e.g. headless / test contexts).""" + if not deps.web_scan_manager: + return + deps.web_scan_manager.add_scan_completion_callback( + lambda: on_library_scan_completed(deps), + ) diff --git a/core/automation/handlers/quality_scanner.py b/core/automation/handlers/quality_scanner.py new file mode 100644 index 00000000..69ec3f02 --- /dev/null +++ b/core/automation/handlers/quality_scanner.py @@ -0,0 +1,83 @@ +"""Automation handler: ``start_quality_scan`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_start_quality_scan`` closure). Submits the quality scanner +to its executor with the configured scope (default: ``watchlist``) +then polls the shared state dict. +""" + +from __future__ import annotations + +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +_TIMEOUT_SECONDS = 7200 # 2 hours +_POLL_INTERVAL_SECONDS = 3 +_INITIAL_DELAY_SECONDS = 1 + + +def auto_start_quality_scan(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + automation_id = config.get('_automation_id') + state = deps.get_quality_scanner_state() + if state.get('status') == 'running': + return {'status': 'skipped', 'reason': 'Quality scan already running'} + + scope = config.get('scope', 'watchlist') + # Pre-set status before submit so the polling loop doesn't see a + # stale 'finished' from a previous run. + with deps.quality_scanner_lock: + state['status'] = 'running' + deps.quality_scanner_executor.submit(deps.run_quality_scanner, scope, deps.get_current_profile_id()) + deps.update_progress( + automation_id, log_line=f'Quality scan started (scope: {scope})', log_type='info', + ) + + # Monitor progress (max 2 hours). + time.sleep(_INITIAL_DELAY_SECONDS) + poll_start = time.time() + while time.time() - poll_start < _TIMEOUT_SECONDS: + time.sleep(_POLL_INTERVAL_SECONDS) + current_status = state.get('status', 'idle') + if current_status not in ('running',): + break + deps.update_progress( + automation_id, + phase=state.get('phase', 'Scanning...'), + progress=state.get('progress', 0), + processed=state.get('processed', 0), + total=state.get('total', 0), + ) + else: + deps.update_progress( + automation_id, status='error', + phase='Timed out', log_line='Quality scan timed out after 2 hours', + log_type='error', + ) + return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} + + final_status = state.get('status', 'idle') + if final_status == 'error': + err = state.get('error_message', 'Unknown error') + deps.update_progress( + automation_id, status='error', progress=100, + phase='Error', log_line=err, log_type='error', + ) + return {'status': 'error', 'reason': err, '_manages_own_progress': True} + + issues = state.get('low_quality', 0) + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', + log_line=f'Quality scan complete — {issues} issues found', + log_type='success', + ) + return { + 'status': 'completed', 'scope': scope, '_manages_own_progress': True, + 'tracks_scanned': state.get('processed', 0), + 'quality_met': state.get('quality_met', 0), + 'low_quality': issues, + 'matched': state.get('matched', 0), + } diff --git a/core/automation/handlers/refresh_mirrored.py b/core/automation/handlers/refresh_mirrored.py new file mode 100644 index 00000000..76ea32ce --- /dev/null +++ b/core/automation/handlers/refresh_mirrored.py @@ -0,0 +1,308 @@ +"""Automation handler: ``refresh_mirrored`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_refresh_mirrored`` closure). Re-pulls track lists from each +mirrored playlist's source (Spotify / Tidal / Deezer / YouTube), +updates the local mirror DB, and emits a ``playlist_changed`` +automation event when the track set actually shifts. + +Source-specific branches (Spotify auth + public-embed fallback, +``spotify_public`` URL→ID resolution, Deezer / Tidal / YouTube) +remain identical to the pre-extraction closure — this is a +mechanical lift, not a redesign. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_refresh_mirrored(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Refresh mirrored playlist(s) from source. + + Returns ``{'status': 'completed', 'refreshed': '', + 'errors': ''}`` on success (counts stringified to match the + automation engine's stat-rendering convention). + """ + db = deps.get_database() + playlist_id = config.get('playlist_id') + refresh_all = config.get('all', False) + auto_id = config.get('_automation_id') + + if refresh_all: + playlists = db.get_mirrored_playlists() + elif playlist_id: + p = db.get_mirrored_playlist(int(playlist_id)) + playlists = [p] if p else [] + else: + return {'status': 'error', 'reason': 'No playlist specified'} + + # Filter out sources that can't be refreshed (no external API). + playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] + + refreshed = 0 + errors = [] + for idx, pl in enumerate(playlists): + try: + source = pl.get('source', '') + source_id = pl.get('source_playlist_id', '') + deps.update_progress( + auto_id, + progress=(idx / max(1, len(playlists))) * 100, + phase=f'Refreshing: "{pl.get("name", "")}"', + current_item=pl.get('name', ''), + ) + tracks = None + + if source == 'spotify': + # Try authenticated API first, fall back to public embed scraper. + if deps.spotify_client and deps.spotify_client.is_spotify_authenticated(): + playlist_obj = deps.spotify_client.get_playlist_by_id(source_id) + if playlist_obj and playlist_obj.tracks: + tracks = [] + for t in playlist_obj.tracks: + artist_name = t.artists[0] if t.artists else '' + track_dict = { + 'track_name': t.name or '', + 'artist_name': str(artist_name), + 'album_name': t.album or '', + 'duration_ms': t.duration_ms or 0, + 'source_track_id': t.id or '', + } + # Spotify data IS official — auto-mark as discovered. + if t.id: + _album_obj = {'name': t.album or ''} + if getattr(t, 'image_url', None): + _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] + track_dict['extra_data'] = json.dumps({ + 'discovered': True, + 'provider': 'spotify', + 'confidence': 1.0, + 'matched_data': { + 'id': t.id, + 'name': t.name or '', + 'artists': [{'name': str(a)} for a in (t.artists or [])], + 'album': _album_obj, + 'duration_ms': t.duration_ms or 0, + 'image_url': getattr(t, 'image_url', None), + } + }) + tracks.append(track_dict) + + # Fallback: public embed scraper (no auth needed). + if tracks is None: + try: + from core.spotify_public_scraper import scrape_spotify_embed + embed_data = scrape_spotify_embed('playlist', source_id) + if embed_data and not embed_data.get('error') and embed_data.get('tracks'): + embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' + tracks = [] + for t in embed_data['tracks']: + artist_names = [a['name'] for a in t.get('artists', [])] + artist_name = artist_names[0] if artist_names else '' + track_dict = { + 'track_name': t.get('name', ''), + 'artist_name': artist_name, + 'album_name': embed_album, + 'duration_ms': t.get('duration_ms', 0), + 'source_track_id': t.get('id', ''), + } + # Store Spotify track ID hint but don't mark discovered — + # Discover step needs to run for proper album art. + if t.get('id'): + track_dict['extra_data'] = json.dumps({ + 'discovered': False, + 'spotify_hint': { + 'id': t['id'], + 'name': t.get('name', ''), + 'artists': t.get('artists', []), + } + }) + tracks.append(track_dict) + except Exception as e: + deps.logger.warning(f"Spotify public scraper fallback failed for {source_id}: {e}") + + elif source == 'spotify_public': + # source_playlist_id is an MD5 hash; extract actual Spotify ID from stored description (URL). + try: + from core.spotify_public_scraper import parse_spotify_url, scrape_spotify_embed + spotify_url = pl.get('description', '') + parsed = parse_spotify_url(spotify_url) if spotify_url else None + + # If Spotify is authenticated, use the full API (auto-discovers with album art). + if (parsed and parsed.get('type') == 'playlist' + and deps.spotify_client and deps.spotify_client.is_spotify_authenticated()): + playlist_obj = deps.spotify_client.get_playlist_by_id(parsed['id']) + if playlist_obj and playlist_obj.tracks: + tracks = [] + for t in playlist_obj.tracks: + artist_name = t.artists[0] if t.artists else '' + track_dict = { + 'track_name': t.name or '', + 'artist_name': str(artist_name), + 'album_name': t.album or '', + 'duration_ms': t.duration_ms or 0, + 'source_track_id': t.id or '', + } + if t.id: + _album_obj = {'name': t.album or ''} + if getattr(t, 'image_url', None): + _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] + track_dict['extra_data'] = json.dumps({ + 'discovered': True, + 'provider': 'spotify', + 'confidence': 1.0, + 'matched_data': { + 'id': t.id, + 'name': t.name or '', + 'artists': [{'name': str(a)} for a in (t.artists or [])], + 'album': _album_obj, + 'duration_ms': t.duration_ms or 0, + 'image_url': getattr(t, 'image_url', None), + } + }) + tracks.append(track_dict) + + # Fallback: public embed scraper (no auth or album-type URL). + if tracks is None and parsed: + embed_data = scrape_spotify_embed(parsed['type'], parsed['id']) + if embed_data and not embed_data.get('error') and embed_data.get('tracks'): + embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' + tracks = [] + for t in embed_data['tracks']: + artist_names = [a['name'] for a in t.get('artists', [])] + artist_name = artist_names[0] if artist_names else '' + tracks.append({ + 'track_name': t.get('name', ''), + 'artist_name': artist_name, + 'album_name': embed_album, + 'duration_ms': t.get('duration_ms', 0), + 'source_track_id': t.get('id', ''), + }) + # No extra_data — let preservation code keep existing discovery data. + except Exception as e: + deps.logger.warning(f"Spotify public playlist refresh failed for {source_id}: {e}") + + elif source == 'deezer': + try: + deezer = deps.get_deezer_client() + playlist_data = deezer.get_playlist(source_id) + if playlist_data and playlist_data.get('tracks'): + tracks = [] + for t in playlist_data['tracks']: + artist_name = t['artists'][0] if t.get('artists') else '' + tracks.append({ + 'track_name': t.get('name', ''), + 'artist_name': str(artist_name), + 'album_name': t.get('album', ''), + 'duration_ms': t.get('duration_ms', 0), + 'source_track_id': str(t.get('id', '')), + }) + except Exception as e: + deps.logger.warning(f"Deezer playlist refresh failed for {source_id}: {e}") + + elif source == 'tidal': + if not deps.tidal_client or not deps.tidal_client.is_authenticated(): + deps.logger.warning(f"Tidal not authenticated — skipping refresh for '{pl.get('name', '')}'") + deps.update_progress( + auto_id, + log_line=f'Skipped "{pl.get("name", "")}" — Tidal not authenticated', + log_type='skip', + ) + continue + full_playlist = deps.tidal_client.get_playlist(source_id) + if full_playlist and full_playlist.tracks: + tracks = [] + for t in full_playlist.tracks: + artist_name = t.artists[0] if t.artists else '' + tracks.append({ + 'track_name': t.name or '', + 'artist_name': str(artist_name), + 'album_name': t.album or '', + 'duration_ms': t.duration_ms or 0, + 'source_track_id': t.id or '', + }) + + elif source == 'youtube': + # source_playlist_id is now a deterministic hash; use stored description (original URL) for refresh. + yt_url = pl.get('description', '') or f"https://www.youtube.com/playlist?list={source_id}" + playlist_data = deps.parse_youtube_playlist(yt_url) + if playlist_data and playlist_data.get('tracks'): + tracks = [] + for t in playlist_data['tracks']: + artist_name = t['artists'][0] if t.get('artists') else '' + tracks.append({ + 'track_name': t.get('name', ''), + 'artist_name': str(artist_name), + 'album_name': '', + 'duration_ms': t.get('duration_ms', 0), + 'source_track_id': t.get('id', ''), + }) + + if tracks is not None: + # Compare old vs new track IDs to detect changes. + old_tracks = db.get_mirrored_playlist_tracks(pl['id']) if pl.get('id') else [] + old_ids = {t.get('source_track_id') for t in old_tracks if t.get('source_track_id')} + new_ids = {t.get('source_track_id') for t in tracks if t.get('source_track_id')} + + # Preserve existing discovery extra_data for tracks that still exist. + old_extra_map = db.get_mirrored_tracks_extra_data_map(pl['id']) if pl.get('id') else {} + for t in tracks: + sid = t.get('source_track_id', '') + if sid and sid in old_extra_map and 'extra_data' not in t: + t['extra_data'] = old_extra_map[sid] + + db.mirror_playlist( + source=source, + source_playlist_id=source_id, + name=pl['name'], + tracks=tracks, + profile_id=pl.get('profile_id', 1), + owner=pl.get('owner'), + image_url=pl.get('image_url'), + ) + refreshed += 1 + + # Emit playlist_changed if tracks actually changed. + if old_ids != new_ids: + added_count = len(new_ids - old_ids) + removed_count = len(old_ids - new_ids) + deps.logger.info( + f"[AUTOMATION] Playlist changed: '{pl.get('name', '')}' — " + f"{added_count} added, {removed_count} removed (old={len(old_ids)}, new={len(new_ids)})" + ) + deps.update_progress( + auto_id, + log_line=f'"{pl.get("name", "")}" — {added_count} added, {removed_count} removed', + log_type='success', + ) + try: + if deps.engine: + deps.engine.emit('playlist_changed', { + 'playlist_name': pl.get('name', ''), + 'playlist_id': str(pl.get('id', '')), + 'old_count': str(len(old_ids)), + 'new_count': str(len(new_ids)), + 'added': str(added_count), + 'removed': str(removed_count), + }) + except Exception as e: + deps.logger.debug("playlist_synced automation emit failed: %s", e) + else: + deps.logger.warning(f"[AUTOMATION] No changes: '{pl.get('name', '')}' (tracks={len(old_ids)})") + deps.update_progress( + auto_id, + log_line=f'No changes: "{pl.get("name", "")}"', + log_type='skip', + ) + except Exception as e: + errors.append(f"{pl.get('name', '?')}: {str(e)}") + deps.update_progress( + auto_id, + log_line=f'Error: {pl.get("name", "?")} — {str(e)}', + log_type='error', + ) + return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors))} diff --git a/core/automation/handlers/registration.py b/core/automation/handlers/registration.py new file mode 100644 index 00000000..94f23db6 --- /dev/null +++ b/core/automation/handlers/registration.py @@ -0,0 +1,184 @@ +"""One-stop registration of every extracted automation handler. + +``web_server`` builds the deps once at startup and calls +:func:`register_all` here. Each new handler module gets one line in +this file when it lands. +""" + +from __future__ import annotations + +from core.automation.deps import AutomationDeps +from core.automation.handlers.process_wishlist import auto_process_wishlist +from core.automation.handlers.scan_watchlist import auto_scan_watchlist +from core.automation.handlers.scan_library import auto_scan_library +from core.automation.handlers.refresh_mirrored import auto_refresh_mirrored +from core.automation.handlers.sync_playlist import auto_sync_playlist +from core.automation.handlers.discover_playlist import auto_discover_playlist +from core.automation.handlers.playlist_pipeline import auto_playlist_pipeline +from core.automation.handlers.personalized_pipeline import auto_personalized_pipeline +from core.automation.handlers.database_update import ( + auto_start_database_update, auto_deep_scan_library, +) +from core.automation.handlers.duplicate_cleaner import auto_run_duplicate_cleaner +from core.automation.handlers.quality_scanner import auto_start_quality_scan +from core.automation.handlers.maintenance import ( + auto_clear_quarantine, + auto_cleanup_wishlist, + auto_update_discovery_pool, + auto_backup_database, + auto_refresh_beatport_cache, +) +from core.automation.handlers.download_cleanup import ( + auto_clean_search_history, + auto_clean_completed_downloads, + auto_full_cleanup, +) +from core.automation.handlers.run_script import auto_run_script +from core.automation.handlers.search_and_download import auto_search_and_download +from core.automation.handlers.progress_callbacks import ( + progress_init, + progress_finish, + record_history, + register_library_scan_completed_emitter, +) + + +def register_all(deps: AutomationDeps) -> None: + """Wire every extracted handler to the engine. + + Each ``register_action_handler`` call binds the action name (the + string the trigger uses to look up its action) to a thin lambda + that injects ``deps`` and forwards the engine-supplied config. + Guards stay alongside their handler so duplicate-run prevention + behaves identically to the pre-extraction code. + """ + engine = deps.engine + + # Self-guards prevent duplicate runs of the SAME operation, but + # different operations can run concurrently — wishlist downloads + # use bandwidth, watchlist scans use API calls, library scans use + # media-server CPU. Different resources, no contention. + engine.register_action_handler( + 'process_wishlist', + lambda config: auto_process_wishlist(config, deps), + guard_fn=deps.is_wishlist_actually_processing, + ) + engine.register_action_handler( + 'scan_watchlist', + lambda config: auto_scan_watchlist(config, deps), + guard_fn=deps.is_watchlist_actually_scanning, + ) + engine.register_action_handler( + 'scan_library', + lambda config: auto_scan_library(config, deps), + deps.state.is_scan_library_active, + ) + + # Playlist lifecycle handlers. The pipeline composes refresh + + # sync + discover (it imports them directly), so all four ship + # together. The pipeline guard prevents an in-flight pipeline + # from being re-triggered mid-run. + engine.register_action_handler( + 'refresh_mirrored', + lambda config: auto_refresh_mirrored(config, deps), + ) + engine.register_action_handler( + 'sync_playlist', + lambda config: auto_sync_playlist(config, deps), + ) + engine.register_action_handler( + 'discover_playlist', + lambda config: auto_discover_playlist(config, deps), + ) + engine.register_action_handler( + 'playlist_pipeline', + lambda config: auto_playlist_pipeline(config, deps), + deps.state.is_pipeline_running, + ) + # Personalized pipeline shares the pipeline_running flag with the + # mirrored pipeline so the two can't overlap (single sync queue, + # single wishlist worker). + engine.register_action_handler( + 'personalized_pipeline', + lambda config: auto_personalized_pipeline(config, deps), + deps.state.is_pipeline_running, + ) + + # Database update + deep scan share the db_update_state guard — + # only one operation can mutate that state at a time. + engine.register_action_handler( + 'start_database_update', + lambda config: auto_start_database_update(config, deps), + lambda: deps.get_db_update_state().get('status') == 'running', + ) + engine.register_action_handler( + 'deep_scan_library', + lambda config: auto_deep_scan_library(config, deps), + lambda: deps.get_db_update_state().get('status') == 'running', + ) + engine.register_action_handler( + 'run_duplicate_cleaner', + lambda config: auto_run_duplicate_cleaner(config, deps), + lambda: deps.get_duplicate_cleaner_state().get('status') == 'running', + ) + engine.register_action_handler( + 'clear_quarantine', + lambda config: auto_clear_quarantine(config, deps), + ) + engine.register_action_handler( + 'cleanup_wishlist', + lambda config: auto_cleanup_wishlist(config, deps), + ) + engine.register_action_handler( + 'update_discovery_pool', + lambda config: auto_update_discovery_pool(config, deps), + ) + engine.register_action_handler( + 'start_quality_scan', + lambda config: auto_start_quality_scan(config, deps), + lambda: deps.get_quality_scanner_state().get('status') == 'running', + ) + engine.register_action_handler( + 'backup_database', + lambda config: auto_backup_database(config, deps), + ) + engine.register_action_handler( + 'refresh_beatport_cache', + lambda config: auto_refresh_beatport_cache(config, deps), + ) + engine.register_action_handler( + 'clean_search_history', + lambda config: auto_clean_search_history(config, deps), + ) + engine.register_action_handler( + 'clean_completed_downloads', + lambda config: auto_clean_completed_downloads(config, deps), + ) + engine.register_action_handler( + 'full_cleanup', + lambda config: auto_full_cleanup(config, deps), + ) + engine.register_action_handler( + 'run_script', + lambda config: auto_run_script(config, deps), + ) + engine.register_action_handler( + 'search_and_download', + lambda config: auto_search_and_download(config, deps), + ) + + # Progress + history callbacks: the engine invokes these around + # each handler run. Lift the closures from + # `web_server._register_automation_handlers` into thin lambdas + # that delegate into the extracted top-level functions. + engine.register_progress_callbacks( + lambda aid, name, action_type: progress_init(aid, name, action_type, deps), + lambda aid, result: progress_finish(aid, result, deps), + deps.update_progress, + lambda aid, result: record_history(aid, result, deps), + ) + + # `library_scan_completed` event: when the media-server scan + # manager finishes a scan, emit the event so any automation can + # trigger off it. No-op when no scan manager is configured. + register_library_scan_completed_emitter(deps) diff --git a/core/automation/handlers/run_script.py b/core/automation/handlers/run_script.py new file mode 100644 index 00000000..2d10a6fb --- /dev/null +++ b/core/automation/handlers/run_script.py @@ -0,0 +1,103 @@ +"""Automation handler: ``run_script`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_run_script`` closure). Runs a user-provided shell or Python +script from the configured scripts directory with bounded timeout + +captured stdout/stderr. Path-traversal guard ensures users can't +escape the scripts directory. + +Environment variables exposed to the script: +- ``SOULSYNC_EVENT``: triggering event type (when fired by an event) +- ``SOULSYNC_AUTOMATION``: automation name +- ``SOULSYNC_SCRIPTS_DIR``: absolute path to the scripts dir +""" + +from __future__ import annotations + +import os +import subprocess as _sp +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +_MAX_TIMEOUT_SECONDS = 300 # Hard cap on user-supplied timeout config. + + +def auto_run_script(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + script_name = config.get('script_name', '') + timeout = min(int(config.get('timeout', 60)), _MAX_TIMEOUT_SECONDS) + automation_id = config.get('_automation_id') + + if not script_name: + return {'status': 'error', 'error': 'No script selected'} + + scripts_dir = deps.docker_resolve_path(deps.config_manager.get('scripts.path', './scripts')) + if not scripts_dir or not os.path.isdir(scripts_dir): + os.makedirs(scripts_dir, exist_ok=True) + return { + 'status': 'error', + 'error': 'Scripts directory is empty. Add scripts to the scripts/ folder.', + } + + script_path = os.path.join(scripts_dir, script_name) + script_path = os.path.realpath(script_path) + + # Security: block path traversal — script must resolve under + # the scripts dir, no symlinks/.. tricks allowed out. + if not script_path.startswith(os.path.realpath(scripts_dir)): + return {'status': 'error', 'error': 'Script path traversal blocked'} + + if not os.path.isfile(script_path): + return {'status': 'error', 'error': f'Script not found: {script_name}'} + + deps.update_progress(automation_id, phase=f'Running {script_name}...', progress=10) + + # Build environment with SoulSync context. + env = os.environ.copy() + event_data = config.get('_event_data') or {} + env['SOULSYNC_EVENT'] = str(event_data.get('type', '')) + env['SOULSYNC_AUTOMATION'] = config.get('_automation_name', '') + env['SOULSYNC_SCRIPTS_DIR'] = scripts_dir + + try: + # Determine how to run the script. + if script_path.endswith('.py'): + cmd = ['python', script_path] + elif script_path.endswith('.sh'): + cmd = ['bash', script_path] + else: + cmd = [script_path] + + result = _sp.run( + cmd, + capture_output=True, text=True, timeout=timeout, + cwd=scripts_dir, env=env, + ) + + deps.update_progress(automation_id, phase='Script completed', progress=100) + + stdout = result.stdout[:2000] if result.stdout else '' + stderr = result.stderr[:1000] if result.stderr else '' + + if result.returncode == 0: + deps.logger.info(f"Script '{script_name}' completed (exit 0)") + else: + deps.logger.warning(f"Script '{script_name}' exited with code {result.returncode}") + + return { + 'status': 'completed' if result.returncode == 0 else 'error', + 'exit_code': str(result.returncode), + 'stdout': stdout, + 'stderr': stderr, + 'script': script_name, + } + except _sp.TimeoutExpired: + deps.update_progress(automation_id, phase='Script timed out', progress=100) + return { + 'status': 'error', + 'error': f'Script timed out after {timeout}s', + 'script': script_name, + } + except Exception as e: # noqa: BLE001 — automation handlers must never raise + return {'status': 'error', 'error': str(e), 'script': script_name} diff --git a/core/automation/handlers/scan_library.py b/core/automation/handlers/scan_library.py new file mode 100644 index 00000000..824d293f --- /dev/null +++ b/core/automation/handlers/scan_library.py @@ -0,0 +1,158 @@ +"""Automation handler: ``scan_library`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_scan_library`` closure). The handler triggers a media-server +scan via ``web_scan_manager``, then polls the manager's status until +the scan completes (or a 30-minute timeout fires). Progress phases +are emitted via :func:`AutomationDeps.update_progress` so the +trigger card stays current throughout the run. + +The handler manages its own progress reporting (it sets +``_manages_own_progress: True`` in the result) so the engine doesn't +overwrite the live phase string with a generic 'completed' label. +""" + +from __future__ import annotations + +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +# Outer poll cap — covers extreme worst case (long Plex scans on +# huge libraries). Past this point we surface a clear timeout error +# so users notice rather than letting the trigger hang forever. +_SCAN_TIMEOUT_SECONDS = 1800 + +# Per-phase poll intervals. +_POLL_SCHEDULED_SECONDS = 2 +_POLL_SCANNING_SECONDS = 5 +_POLL_UNKNOWN_SECONDS = 2 + +# Progress percentage waypoints. +_PROGRESS_SCHEDULED_MAX = 14 +_PROGRESS_SCAN_START = 15 +_PROGRESS_SCAN_MAX = 95 + + +def auto_scan_library(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run a media-server library scan and stream progress to the + trigger card. + + Returns one of: + - ``{'status': 'completed', '_manages_own_progress': True, ...}`` + - ``{'status': 'skipped', 'reason': 'Scan already being tracked'}`` + - ``{'status': 'error', 'reason': '...', '_manages_own_progress': True}`` + """ + automation_id = config.get('_automation_id') + + if not deps.web_scan_manager: + return {'status': 'error', 'reason': 'Scan manager not available'} + + # If another automation is already tracking the scan, just forward + # the request — the original tracker keeps emitting progress. + if deps.state.is_scan_library_active(): + deps.web_scan_manager.request_scan('Automation trigger (additional batch)') + return {'status': 'skipped', 'reason': 'Scan already being tracked'} + + deps.state.set_scan_library_id(automation_id) + + try: + result = deps.web_scan_manager.request_scan('Automation trigger') + scan_status_val = result.get('status', 'unknown') + + if scan_status_val == 'queued': + deps.update_progress( + automation_id, + log_line='Scan already in progress — waiting for completion', + log_type='info', + ) + else: + delay = result.get('delay_seconds', 60) + deps.update_progress( + automation_id, + log_line=f'Scan scheduled (debounce: {delay}s)', + log_type='info', + ) + + # Unified polling loop — handles debounce → scanning → idle. + poll_start = time.time() + scan_started = (scan_status_val == 'queued') + while time.time() - poll_start < _SCAN_TIMEOUT_SECONDS: + status = deps.web_scan_manager.get_scan_status() + st = status.get('status') + + if st == 'idle': + break # Scan completed (or finished before we polled) + + if st == 'scheduled': + elapsed = int(time.time() - poll_start) + deps.update_progress( + automation_id, + phase=f'Waiting for scan to start... ({elapsed}s)', + progress=min(int(elapsed / 60 * 10), _PROGRESS_SCHEDULED_MAX), + ) + time.sleep(_POLL_SCHEDULED_SECONDS) + continue + + if st == 'scanning': + if not scan_started: + scan_started = True + deps.update_progress( + automation_id, + progress=_PROGRESS_SCAN_START, + log_line='Scan triggered on media server', + log_type='success', + ) + elapsed = status.get('elapsed_seconds', 0) + max_time = status.get('max_time_seconds', 300) + pct = min(_PROGRESS_SCAN_START + int(elapsed / max_time * 80), _PROGRESS_SCAN_MAX) + mins, secs = divmod(elapsed, 60) + deps.update_progress( + automation_id, + phase=f'Library scan in progress... ({mins}m {secs}s)', + progress=pct, + ) + time.sleep(_POLL_SCANNING_SECONDS) + continue + + time.sleep(_POLL_UNKNOWN_SECONDS) + else: + # 30-min timeout reached + deps.update_progress( + automation_id, + status='error', + phase='Timed out', + log_line='Library scan timed out after 30 minutes', + log_type='error', + ) + return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} + + elapsed = round(time.time() - poll_start, 1) + deps.update_progress( + automation_id, + status='finished', + progress=100, + phase='Complete', + log_line='Library scan completed', + log_type='success', + ) + return { + 'status': 'completed', + '_manages_own_progress': True, + 'scan_duration_seconds': elapsed, + } + + except Exception as e: # noqa: BLE001 — automation handlers must never raise into the engine + deps.update_progress( + automation_id, + status='error', + phase='Error', + log_line=str(e), + log_type='error', + ) + return {'status': 'error', 'error': str(e), '_manages_own_progress': True} + + finally: + deps.state.set_scan_library_id(None) diff --git a/core/automation/handlers/scan_watchlist.py b/core/automation/handlers/scan_watchlist.py new file mode 100644 index 00000000..fe522f62 --- /dev/null +++ b/core/automation/handlers/scan_watchlist.py @@ -0,0 +1,46 @@ +"""Automation handler: ``scan_watchlist`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_scan_watchlist`` closure). The watchlist scanner returns +summary stats for the trigger card only when a fresh scan actually +ran — detected by snapshotting ``id(state_dict)`` before/after, since +the live processor reassigns the dict on each new scan. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_scan_watchlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run a watchlist scan when the automation triggers. + + Pre-scan we capture ``id(watchlist_scan_state)`` so we can tell + afterwards whether the worker ran (and reassigned the state dict) + or short-circuited (kept the same dict). Only fresh scans report + summary stats — repeat triggers without an intervening run return + a bare ``completed``. + """ + try: + pre_state = deps.get_watchlist_scan_state() + pre_state_id = id(pre_state) + deps.process_watchlist_scan_automatically( + automation_id=config.get('_automation_id'), + profile_id=config.get('_profile_id'), + ) + post_state = deps.get_watchlist_scan_state() + # Fresh scan = state dict was reassigned mid-run. + if id(post_state) != pre_state_id: + summary = post_state.get('summary', {}) if isinstance(post_state, dict) else {} + return { + 'status': 'completed', + 'artists_scanned': summary.get('total_artists', 0), + 'successful_scans': summary.get('successful_scans', 0), + 'new_tracks_found': summary.get('new_tracks_found', 0), + 'tracks_added_to_wishlist': summary.get('tracks_added_to_wishlist', 0), + } + return {'status': 'completed'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise into the engine + return {'status': 'error', 'error': str(e)} diff --git a/core/automation/handlers/search_and_download.py b/core/automation/handlers/search_and_download.py new file mode 100644 index 00000000..fa49c534 --- /dev/null +++ b/core/automation/handlers/search_and_download.py @@ -0,0 +1,57 @@ +"""Automation handler: ``search_and_download`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_search_and_download`` closure). Searches for a track by +name/artist string and dispatches the best match through the +download orchestrator. Query can come from the trigger config +(direct value) or from event data (e.g. webhook payload). +""" + +from __future__ import annotations + +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_search_and_download(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + automation_id = config.get('_automation_id') + query = config.get('query', '').strip() + # Event-triggered: pull query from event data (e.g. webhook_received). + if not query: + event_data = config.get('_event_data', {}) + query = (event_data.get('query', '') or '').strip() + if not query: + if automation_id: + deps.update_progress( + automation_id, log_line='No search query provided', log_type='error', + ) + return {'status': 'error', 'error': 'No search query provided'} + try: + if automation_id: + deps.update_progress( + automation_id, phase='Searching', + log_line=f'Searching: {query}', log_type='info', + ) + result = deps.run_async(deps.download_orchestrator.search_and_download_best(query)) + if result: + if automation_id: + deps.update_progress( + automation_id, + log_line=f'Download started for: {query}', + log_type='success', + ) + return {'status': 'completed', 'query': query, 'download_id': result} + if automation_id: + deps.update_progress( + automation_id, + log_line=f'No match found for: {query}', + log_type='warning', + ) + return {'status': 'not_found', 'query': query, 'error': 'No match found'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise + if automation_id: + deps.update_progress( + automation_id, log_line=f'Error: {e}', log_type='error', + ) + return {'status': 'error', 'query': query, 'error': str(e)} diff --git a/core/automation/handlers/sync_playlist.py b/core/automation/handlers/sync_playlist.py new file mode 100644 index 00000000..5734b48f --- /dev/null +++ b/core/automation/handlers/sync_playlist.py @@ -0,0 +1,195 @@ +"""Automation handler: ``sync_playlist`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_sync_playlist`` closure). Syncs a mirrored playlist to the +configured media server, using discovered metadata when available +and skipping undiscovered tracks. When triggered on a schedule with +no track changes since the last sync, short-circuits with +``status: skipped`` (saves Plex / Jellyfin / Navidrome from +needless rewrites).""" + +from __future__ import annotations + +import hashlib +import json +import threading +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_sync_playlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Sync a mirrored playlist to the active media server. + + Behavior: + - Tracks with discovered metadata (extra_data.discovered + matched_data) + are routed via the official metadata. + - Tracks with a Spotify hint (real Spotify ID from the embed + scraper) are included so they can still hit Soulseek + the + wishlist. + - Tracks with neither are counted as ``skipped_tracks``. + - Empty result → ``status: skipped`` with the skipped count. + - Same track set as last sync (matched_tracks unchanged) → + ``status: skipped`` (no-op). + - Otherwise spawns a daemon thread running ``run_sync_task`` and + returns ``status: started`` with ``_manages_own_progress: True``. + """ + auto_id = config.get('_automation_id') + playlist_id = config.get('playlist_id') + if not playlist_id: + return {'status': 'error', 'reason': 'No playlist specified'} + + db = deps.get_database() + pl = db.get_mirrored_playlist(int(playlist_id)) + if not pl: + return {'status': 'error', 'reason': 'Playlist not found'} + + tracks = db.get_mirrored_playlist_tracks(int(playlist_id)) + if not tracks: + return {'status': 'error', 'reason': 'No tracks in playlist'} + + # Convert mirrored tracks to format expected by run_sync_task. + # Use discovered metadata when available, fall back to Spotify + # hint or raw playlist fields when not. + tracks_json = [] + skipped_count = 0 + + for t in tracks: + # Parse extra_data for discovery info. + extra = {} + if t.get('extra_data'): + try: + extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] + except (json.JSONDecodeError, TypeError): + pass + + if extra.get('discovered') and extra.get('matched_data'): + # Use official discovered metadata. + md = extra['matched_data'] + album_raw = md.get('album', '') + album_obj = album_raw if isinstance(album_raw, dict) else {'name': album_raw or ''} + _track_entry = { + 'name': md.get('name', ''), + 'artists': md.get('artists', [{'name': t.get('artist_name', '')}]), + 'album': album_obj, + 'duration_ms': md.get('duration_ms', 0), + 'id': md.get('id', ''), + } + if md.get('track_number'): + _track_entry['track_number'] = md['track_number'] + if md.get('disc_number'): + _track_entry['disc_number'] = md['disc_number'] + tracks_json.append(_track_entry) + else: + # NOT discovered — try to include using available metadata so + # the track can still be searched on Soulseek and added to + # wishlist. Without this, failed discovery blocks the entire + # download pipeline. + # + # Priority: spotify_hint (has real Spotify ID from embed + # scraper) > raw playlist fields (only if source_track_id + # is valid). + hint = extra.get('spotify_hint', {}) + # Build album object with cover art from the mirrored playlist track. + track_image = (t.get('image_url') or '').strip() + album_obj = { + 'name': (t.get('album_name') or '').strip(), + 'images': [{'url': track_image, 'height': 300, 'width': 300}] if track_image else [], + } + + if hint.get('id') and hint.get('name'): + # spotify_hint has proper Spotify track ID + metadata from embed scraper. + hint_artists = hint.get('artists', []) + if hint_artists and isinstance(hint_artists[0], str): + hint_artists = [{'name': a} for a in hint_artists] + elif hint_artists and isinstance(hint_artists[0], dict): + pass # Already in correct format + else: + hint_artists = [{'name': t.get('artist_name', '')}] + tracks_json.append({ + 'name': hint['name'], + 'artists': hint_artists, + 'album': album_obj, + 'duration_ms': t.get('duration_ms', 0), + 'id': hint['id'], + }) + elif t.get('source_track_id') and (t.get('track_name') or '').strip(): + # Has a valid source ID and track name — usable for wishlist. + tracks_json.append({ + 'name': t['track_name'].strip(), + 'artists': [{'name': (t.get('artist_name') or '').strip() or 'Unknown Artist'}], + 'album': album_obj, + 'duration_ms': t.get('duration_ms', 0), + 'id': t['source_track_id'], + }) + else: + skipped_count += 1 # No usable ID or name — truly can't process. + + if not tracks_json: + deps.update_progress( + auto_id, + log_line=f'No discovered tracks — {skipped_count} need discovery first', + log_type='skip', + ) + return { + 'status': 'skipped', + 'reason': f'No discovered tracks to sync ({skipped_count} tracks need discovery first)', + 'skipped_tracks': str(skipped_count), + } + + # Preflight: hash the track list and compare against last sync. + # Skip if the exact same set of tracks was already synced and + # everything matched (no-op preserves Plex / Jellyfin / Navidrome + # from needless rewrites). + track_ids_str = ','.join(sorted(t.get('id', '') for t in tracks_json)) + tracks_hash = hashlib.md5(track_ids_str.encode()).hexdigest() + + sync_id_key = f"auto_mirror_{playlist_id}" + try: + sync_statuses = deps.load_sync_status_file() + last_status = sync_statuses.get(sync_id_key, {}) + last_hash = last_status.get('tracks_hash', '') + last_matched = last_status.get('matched_tracks', -1) + + if last_hash == tracks_hash and last_matched >= len(tracks_json): + # Exact same tracks, all matched last time — nothing to do. + deps.update_progress( + auto_id, + log_line=f'All {len(tracks_json)} tracks unchanged since last sync — skipping', + log_type='skip', + ) + return { + 'status': 'skipped', + 'reason': f'All {len(tracks_json)} tracks unchanged since last sync', + } + except Exception as e: + deps.logger.debug("mirror sync last-status read: %s", e) + + deps.update_progress( + auto_id, + progress=50, + phase=f'Syncing "{pl["name"]}"', + log_line=f'{len(tracks_json)} discovered, {skipped_count} skipped', + log_type='info', + ) + + sync_id = f"auto_mirror_{playlist_id}" + deps.update_progress( + auto_id, + progress=90, + log_line=f'Starting sync: {len(tracks_json)} tracks', + log_type='success', + ) + threading.Thread( + target=deps.run_sync_task, + args=(sync_id, pl['name'], tracks_json, auto_id, 1, pl.get('image_url', '')), + daemon=True, + name=f'auto-sync-{playlist_id}', + ).start() + return { + 'status': 'started', + 'playlist_name': pl['name'], + 'discovered_tracks': str(len(tracks_json)), + 'skipped_tracks': str(skipped_count), + '_manages_own_progress': True, + } diff --git a/core/connection_test.py b/core/connection_test.py index 4064bab5..745caa2a 100644 --- a/core/connection_test.py +++ b/core/connection_test.py @@ -191,6 +191,12 @@ def run_service_test(service, test_config): 'tidal': "Tidal download source ready.", 'qobuz': "Qobuz download source ready.", 'hifi': "HiFi download source ready.", + 'deezer_dl': "Deezer download source ready.", + 'amazon': "Amazon download source ready.", + 'lidarr': "Lidarr download source ready.", + 'soundcloud': "SoundCloud download source ready.", + 'torrent': "Torrent download source ready.", + 'usenet': "Usenet download source ready.", 'hybrid': "Download sources ready (Hybrid mode)." } message = mode_messages.get(download_mode, "Download source connected.") @@ -203,6 +209,12 @@ def run_service_test(service, test_config): 'tidal': "Tidal download source not available. Check authentication.", 'qobuz': "Qobuz download source not available. Check authentication.", 'hifi': "HiFi download source not available. Public API instances may be down.", + 'deezer_dl': "Deezer download source not available. Check authentication.", + 'amazon': "Amazon download source not available.", + 'lidarr': "Lidarr download source not available. Check Lidarr URL and API key.", + 'soundcloud': "SoundCloud download source not available.", + 'torrent': "Torrent download source not available. Check Prowlarr and torrent client settings.", + 'usenet': "Usenet download source not available. Check Prowlarr and usenet client settings.", 'hybrid': "Could not connect to download sources. Check configuration." } error = mode_errors.get(download_mode, "Download source connection failed.") @@ -305,6 +317,61 @@ def run_service_test(service, test_config): return False, "Invalid Genius access token." except Exception as e: return False, f"Genius connection error: {str(e)}" + elif service == "usenet_client": + client_type = (config_manager.get('usenet_client.type', '') or '').strip().lower() + url = config_manager.get('usenet_client.url', '') + if not url: + return False, "Usenet client URL is required." + if not client_type: + return False, "Pick a usenet client (SABnzbd or NZBGet)." + try: + from core.usenet_clients import adapter_for_type as _usenet_adapter_for_type + adapter = _usenet_adapter_for_type(client_type) + if adapter is None: + return False, f"Unknown usenet client type: {client_type}" + if not adapter.is_configured(): + if client_type == "sabnzbd": + return False, "SABnzbd needs both URL and API key." + return False, "NZBGet needs URL, username, and password." + if run_async(adapter.check_connection()): + return True, f"Connected to {client_type}" + return False, f"{client_type} probe failed — check URL, credentials, and that the client is running." + except Exception as e: + return False, f"Usenet client connection error: {str(e)}" + elif service == "torrent_client": + client_type = (config_manager.get('torrent_client.type', '') or '').strip().lower() + url = config_manager.get('torrent_client.url', '') + if not url: + return False, "Torrent client URL is required." + if not client_type: + return False, "Pick a torrent client (qBittorrent, Transmission, or Deluge)." + try: + from core.torrent_clients import adapter_for_type + adapter = adapter_for_type(client_type) + if adapter is None: + return False, f"Unknown torrent client type: {client_type}" + if not adapter.is_configured(): + return False, "Torrent client missing required credentials." + if run_async(adapter.check_connection()): + return True, f"Connected to {client_type}" + return False, f"{client_type} probe failed — check URL, credentials, and that the client is running." + except Exception as e: + return False, f"Torrent client connection error: {str(e)}" + elif service == "prowlarr": + url = config_manager.get('prowlarr.url', '') + api_key = config_manager.get('prowlarr.api_key', '') + if not url or not api_key: + return False, "Prowlarr URL and API key are required." + try: + import requests as _req + resp = _req.get(f"{url.rstrip('/')}/api/v1/system/status", + headers={'X-Api-Key': api_key}, timeout=10) + if resp.ok: + version = resp.json().get('version', '?') + return True, f"Connected to Prowlarr v{version}" + return False, f"Prowlarr returned HTTP {resp.status_code}" + except Exception as e: + return False, f"Prowlarr connection error: {str(e)}" elif service == "lidarr" or service == "lidarr_download": url = config_manager.get('lidarr_download.url', '') api_key = config_manager.get('lidarr_download.api_key', '') @@ -379,6 +446,16 @@ def run_service_test(service, test_config): return False, "Hydrabase not connected. Configure URL + API key and click Connect." except Exception as e: return False, f"Hydrabase connection error: {str(e)}" + elif service == "musicbrainz": + try: + from core.metadata.registry import get_musicbrainz_client + mb = get_musicbrainz_client() + results = mb.search_artists("radiohead", limit=1) + if results: + return True, "MusicBrainz reachable" + return False, "MusicBrainz returned no results — may be rate-limited or unreachable." + except Exception as e: + return False, f"MusicBrainz connection error: {str(e)}" elif service == "soundcloud": # Anonymous SoundCloud has no auth, so "test" really means # "is yt-dlp installed and can it reach SoundCloud right now." diff --git a/core/deezer_client.py b/core/deezer_client.py index e564663c..07162228 100644 --- a/core/deezer_client.py +++ b/core/deezer_client.py @@ -89,6 +89,33 @@ def _upgrade_deezer_cover_url(url: str, target_size: int = _DEEZER_MAX_COVER_SIZ return _DEEZER_CDN_SIZE_PATTERN.sub(f'/{target_size}x{target_size}-', url, count=1) +def _is_full_track_payload(payload: Optional[Dict[str, Any]]) -> bool: + """Distinguish a full `/track/` cache hit from partial album-tracks data. + + Three Deezer endpoints feed the per-track cache: + - `/track/` — full record, includes both `track_position` AND + `contributors` (the multi-artist list the contributors-upgrade + path reads). + - `/album//tracks` — partial; includes `track_position` but + omits `contributors`. + - `/search/track` — minimal; lacks `track_position`. + + Pre-fix `get_track_details` only checked `track_position`, so + partial album-tracks payloads were treated as full hits and the + contributors-upgrade silently fell back to single-artist tagging + whenever an album had been fetched before its individual tracks + were post-processed (issue #588). + + `contributors` key presence is the load-bearing distinction — + `[]` is a valid value for genuinely single-artist tracks fetched + via the per-track endpoint, so test for key membership not + truthiness. + """ + if not isinstance(payload, dict): + return False + return 'track_position' in payload and 'contributors' in payload + + # ==================== Dataclasses (match iTunesClient / SpotifyClient format) ==================== @dataclass @@ -546,14 +573,9 @@ class DeezerClient: """Get detailed track info — returns Spotify-compatible dict (metadata source interface)""" cache = get_metadata_cache() cached = cache.get_entity('deezer', 'track', str(track_id)) - if cached and cached.get('title'): - # Search results are cached with minimal data (no track_position). - # Only use cache if it has track_position — the key field from /track/{id}. - # Search results include 'isrc' and 'release_date' but NOT track_position, - # so those fields alone are not sufficient to distinguish full from partial data. - if 'track_position' in cached: - return self._build_enhanced_track(cached) - # Otherwise fall through to fetch full data from API + if cached and cached.get('title') and _is_full_track_payload(cached): + return self._build_enhanced_track(cached) + # Otherwise fall through to fetch full data from API data = self._api_get(f'track/{track_id}') if not data: diff --git a/core/discovery/hero.py b/core/discovery/hero.py index 50800ecb..420de342 100644 --- a/core/discovery/hero.py +++ b/core/discovery/hero.py @@ -17,10 +17,14 @@ logger = logging.getLogger(__name__) def get_current_profile_id() -> int: - """Mirror of web_server.get_current_profile_id — uses Flask g.""" + """Mirror of web_server.get_current_profile_id — uses Flask g. + + Catches RuntimeError too because reading `g` outside a request + context raises that (not AttributeError) — happens when this is + called from background threads (sync, automation, scanners).""" try: return g.profile_id - except AttributeError: + except (AttributeError, RuntimeError): return 1 @@ -92,6 +96,8 @@ def get_discover_hero(): artist_id = artist.spotify_artist_id elif active_source == 'deezer': artist_id = getattr(artist, 'deezer_artist_id', None) or artist.itunes_artist_id + elif active_source == 'musicbrainz': + artist_id = getattr(artist, 'musicbrainz_artist_id', None) or artist.itunes_artist_id else: artist_id = artist.itunes_artist_id if not artist_id: @@ -121,7 +127,7 @@ def get_discover_hero(): valid_artists = list(similar_artists) # FALLBACK: If no valid artists for fallback source, try to resolve IDs on-the-fly - if active_source in ('itunes', 'deezer') and not valid_artists: + if active_source in ('itunes', 'deezer', 'musicbrainz') and not valid_artists: logger.warning(f"[{active_source} Fallback] No artists with {active_source} IDs found, attempting on-the-fly resolution for {len(similar_artists)} artists") resolved_count = 0 for artist in similar_artists: @@ -131,13 +137,20 @@ def get_discover_hero(): continue # Try to resolve ID by name try: - search_results = itunes_client.search_artists(artist.similar_artist_name, limit=1) + resolve_client = itunes_client + if active_source == 'musicbrainz': + from core.metadata.registry import get_musicbrainz_client + resolve_client = get_musicbrainz_client() + search_results = resolve_client.search_artists(artist.similar_artist_name, limit=1) if search_results and len(search_results) > 0: resolved_id = search_results[0].id # Cache the resolved ID for future use if active_source == 'deezer': database.update_similar_artist_deezer_id(artist.id, resolved_id) artist.similar_artist_deezer_id = resolved_id + elif active_source == 'musicbrainz': + database.update_similar_artist_musicbrainz_id(artist.id, resolved_id) + artist.similar_artist_musicbrainz_id = resolved_id else: database.update_similar_artist_itunes_id(artist.id, resolved_id) artist.similar_artist_itunes_id = resolved_id @@ -169,12 +182,15 @@ def get_discover_hero(): artist_id = artist.similar_artist_spotify_id or artist.similar_artist_itunes_id elif active_source == 'deezer': artist_id = getattr(artist, 'similar_artist_deezer_id', None) or artist.similar_artist_itunes_id or artist.similar_artist_spotify_id + elif active_source == 'musicbrainz': + artist_id = getattr(artist, 'similar_artist_musicbrainz_id', None) or artist.similar_artist_itunes_id or artist.similar_artist_spotify_id else: artist_id = artist.similar_artist_itunes_id or artist.similar_artist_spotify_id artist_data = { "spotify_artist_id": artist.similar_artist_spotify_id, "itunes_artist_id": artist.similar_artist_itunes_id, + "musicbrainz_artist_id": getattr(artist, 'similar_artist_musicbrainz_id', None), "artist_id": artist_id, "artist_name": artist.similar_artist_name, "occurrence_count": artist.occurrence_count, @@ -203,11 +219,19 @@ def get_discover_hero(): artist.id, artist_data.get('image_url'), artist_data.get('genres'), artist_data.get('popularity') ) - elif active_source in ('itunes', 'deezer'): - fb_artist_id = getattr(artist, 'similar_artist_deezer_id', None) if active_source == 'deezer' else None - fb_artist_id = fb_artist_id or artist.similar_artist_itunes_id + elif active_source in ('itunes', 'deezer', 'musicbrainz'): + if active_source == 'deezer': + fb_artist_id = getattr(artist, 'similar_artist_deezer_id', None) or artist.similar_artist_itunes_id + fetch_client = itunes_client + elif active_source == 'musicbrainz': + fb_artist_id = getattr(artist, 'similar_artist_musicbrainz_id', None) + from core.metadata.registry import get_musicbrainz_client + fetch_client = get_musicbrainz_client() + else: + fb_artist_id = artist.similar_artist_itunes_id + fetch_client = itunes_client if fb_artist_id: - fb_artist_data = itunes_client.get_artist(fb_artist_id) + fb_artist_data = fetch_client.get_artist(fb_artist_id) if fb_artist_data: artist_data['artist_name'] = fb_artist_data.get('name', artist.similar_artist_name) artist_data['image_url'] = fb_artist_data.get('images', [{}])[0].get('url') if fb_artist_data.get('images') else None diff --git a/core/discovery/qobuz.py b/core/discovery/qobuz.py new file mode 100644 index 00000000..3f08f6ae --- /dev/null +++ b/core/discovery/qobuz.py @@ -0,0 +1,299 @@ +"""Background worker for Qobuz playlist discovery. + +`run_qobuz_discovery_worker(playlist_id, deps)` is the function the +Qobuz discovery start-endpoint submits to its executor to match each +Qobuz playlist track against Spotify (preferred) or the configured +fallback metadata source (iTunes / Deezer / Discogs / MusicBrainz). + +Mirrors `core/discovery/deezer.py` exactly — Qobuz playlists arrive as +dicts (not dataclasses) from `core/qobuz_client.py:get_playlist`, so +this worker uses dict-style access on track data and wraps each entry +in a SimpleNamespace before handing it to the shared +`_search_spotify_for_tidal_track` helper. +""" + +from __future__ import annotations + +import logging +import time +import types +from dataclasses import dataclass +from typing import Any, Callable + +logger = logging.getLogger(__name__) + + +@dataclass +class QobuzDiscoveryDeps: + """Bundle of cross-cutting deps the Qobuz discovery worker needs.""" + qobuz_discovery_states: dict + spotify_client: Any + pause_enrichment_workers: Callable[[str], dict] + resume_enrichment_workers: Callable[[dict, str], None] + get_active_discovery_source: Callable[[], str] + get_metadata_fallback_client: Callable[[], Any] + get_discovery_cache_key: Callable + get_database: Callable[[], Any] + validate_discovery_cache_artist: Callable + search_spotify_for_tidal_track: Callable + build_discovery_wing_it_stub: Callable + add_activity_item: Callable + sync_discovery_results_to_mirrored: Callable + + +def run_qobuz_discovery_worker(playlist_id, deps: QobuzDiscoveryDeps): + """Background worker for Qobuz discovery process (Spotify preferred, fallback metadata source).""" + _ew_state = {} + try: + _ew_state = deps.pause_enrichment_workers('Qobuz discovery') + state = deps.qobuz_discovery_states[playlist_id] + playlist = state['playlist'] + + # Determine which provider to use + discovery_source = deps.get_active_discovery_source() + use_spotify = (discovery_source == 'spotify') and deps.spotify_client and deps.spotify_client.is_spotify_authenticated() + + # Initialize fallback client if needed + itunes_client_instance = None + if not use_spotify: + itunes_client_instance = deps.get_metadata_fallback_client() + + logger.info(f"Starting Qobuz discovery for: {playlist['name']} (using {discovery_source.upper()})") + + # Store discovery source in state for frontend + state['discovery_source'] = discovery_source + + successful_discoveries = 0 + tracks = playlist['tracks'] + + for i, qobuz_track in enumerate(tracks): + if state.get('cancelled', False): + break + + try: + track_name = qobuz_track['name'] + track_artists = qobuz_track['artists'] + track_id = qobuz_track['id'] + track_album = qobuz_track.get('album', '') + track_duration_ms = qobuz_track.get('duration_ms', 0) + + logger.info(f"[{i+1}/{len(tracks)}] Searching {discovery_source.upper()}: {track_name} by {', '.join(track_artists)}") + + # Check discovery cache first + cache_key = deps.get_discovery_cache_key(track_name, track_artists[0] if track_artists else '') + try: + cache_db = deps.get_database() + cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source) + if cached_match and deps.validate_discovery_cache_artist(track_artists[0] if track_artists else '', cached_match): + logger.debug(f"CACHE HIT [{i+1}/{len(tracks)}]: {track_name} by {', '.join(track_artists)}") + cached_artists = cached_match.get('artists', []) + if cached_artists: + cached_artist_str = ', '.join( + a if isinstance(a, str) else a.get('name', '') for a in cached_artists + ) + else: + cached_artist_str = '' + cached_album = cached_match.get('album', '') + if isinstance(cached_album, dict): + cached_album = cached_album.get('name', '') + + result = { + 'qobuz_track': { + 'id': track_id, + 'name': track_name, + 'artists': track_artists or [], + 'album': track_album, + 'duration_ms': track_duration_ms, + }, + 'spotify_data': cached_match, + 'match_data': cached_match, + 'status': 'Found', + 'status_class': 'found', + 'spotify_track': cached_match.get('name', ''), + 'spotify_artist': cached_artist_str, + 'spotify_album': cached_album, + 'spotify_id': cached_match.get('id', ''), + 'discovery_source': discovery_source, + 'index': i + } + successful_discoveries += 1 + state['spotify_matches'] = successful_discoveries + state['discovery_results'].append(result) + state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) + continue + except Exception as cache_err: + logger.error(f"Cache lookup error: {cache_err}") + + # SimpleNamespace duck-type for _search_spotify_for_tidal_track + track_ns = types.SimpleNamespace( + id=track_id, + name=track_name, + artists=track_artists, + album=track_album, + duration_ms=track_duration_ms + ) + + track_result = deps.search_spotify_for_tidal_track( + track_ns, + use_spotify=use_spotify, + itunes_client=itunes_client_instance + ) + + result = { + 'qobuz_track': { + 'id': track_id, + 'name': track_name, + 'artists': track_artists or [], + 'album': track_album, + 'duration_ms': track_duration_ms, + }, + 'spotify_data': None, + 'match_data': None, + 'status': 'Not Found', + 'status_class': 'not-found', + 'spotify_track': '', + 'spotify_artist': '', + 'spotify_album': '', + 'discovery_source': discovery_source + } + + match_confidence = 0.0 + + if use_spotify and isinstance(track_result, tuple): + track_obj, raw_track_data, match_confidence = track_result + album_obj = raw_track_data.get('album', {}) if raw_track_data else {} + if isinstance(album_obj, dict) and not album_obj.get('name') and track_obj.album: + album_obj['name'] = track_obj.album + elif not album_obj and track_obj.album: + album_obj = {'name': track_obj.album} + if isinstance(album_obj, dict) and not album_obj.get('release_date'): + album_obj['release_date'] = getattr(track_obj, 'release_date', '') or '' + _album_images = album_obj.get('images', []) if isinstance(album_obj, dict) else [] + _image_url = _album_images[0].get('url', '') if _album_images else (getattr(track_obj, 'image_url', '') or '') + + match_data = { + 'id': track_obj.id, + 'name': track_obj.name, + 'artists': track_obj.artists, + 'album': album_obj, + 'duration_ms': track_obj.duration_ms, + 'external_urls': track_obj.external_urls, + 'image_url': _image_url, + 'source': 'spotify' + } + if raw_track_data and raw_track_data.get('track_number'): + match_data['track_number'] = raw_track_data['track_number'] + if raw_track_data and raw_track_data.get('disc_number'): + match_data['disc_number'] = raw_track_data['disc_number'] + result['spotify_data'] = match_data + result['match_data'] = match_data + result['status'] = 'Found' + result['status_class'] = 'found' + result['spotify_track'] = track_obj.name + result['spotify_artist'] = ', '.join(track_obj.artists) if isinstance(track_obj.artists, list) else str(track_obj.artists) + result['spotify_album'] = album_obj.get('name', '') if isinstance(album_obj, dict) else str(album_obj) + result['spotify_id'] = track_obj.id + result['confidence'] = match_confidence + successful_discoveries += 1 + state['spotify_matches'] = successful_discoveries + + elif not use_spotify and track_result and isinstance(track_result, dict): + match_confidence = track_result.pop('confidence', 0.80) + match_data = track_result + match_data['source'] = discovery_source + _fb_album = match_data.get('album', {}) + _fb_images = _fb_album.get('images', []) if isinstance(_fb_album, dict) else [] + if _fb_images and 'image_url' not in match_data: + match_data['image_url'] = _fb_images[0].get('url', '') + result['spotify_data'] = match_data + result['match_data'] = match_data + result['status'] = 'Found' + result['status_class'] = 'found' + result['spotify_track'] = match_data.get('name', '') + itunes_artists = match_data.get('artists', []) + result['spotify_artist'] = ', '.join(a if isinstance(a, str) else a.get('name', '') for a in itunes_artists) if itunes_artists else '' + result['spotify_album'] = match_data.get('album', {}).get('name', '') if isinstance(match_data.get('album'), dict) else match_data.get('album', '') + result['spotify_id'] = match_data.get('id', '') + result['confidence'] = match_confidence + successful_discoveries += 1 + state['spotify_matches'] = successful_discoveries + + # Save to discovery cache if match found + if result['status_class'] == 'found' and result.get('match_data'): + try: + cache_db = deps.get_database() + cache_db.save_discovery_cache_match( + cache_key[0], cache_key[1], discovery_source, match_confidence, + result['match_data'], track_name, + track_artists[0] if track_artists else '' + ) + logger.info(f"CACHE SAVED: {track_name} (confidence: {match_confidence:.3f})") + except Exception as cache_err: + logger.error(f"Cache save error: {cache_err}") + + # Auto Wing It fallback for unmatched tracks + if result['status_class'] == 'not-found': + qobuz_t = result.get('qobuz_track', {}) + stub = deps.build_discovery_wing_it_stub( + qobuz_t.get('name', ''), + ', '.join(qobuz_t.get('artists', [])), + qobuz_t.get('duration_ms', 0) + ) + result['status'] = 'Wing It' + result['status_class'] = 'wing-it' + result['spotify_data'] = stub + result['match_data'] = stub + result['spotify_track'] = qobuz_t.get('name', '') + result['spotify_artist'] = ', '.join(qobuz_t.get('artists', [])) + result['wing_it_fallback'] = True + result['confidence'] = 0 + successful_discoveries += 1 + state['spotify_matches'] = successful_discoveries + state['wing_it_count'] = state.get('wing_it_count', 0) + 1 + + result['index'] = i + state['discovery_results'].append(result) + state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) + + time.sleep(0.1) + + except Exception as e: + logger.error(f"Error processing track {i+1}: {e}") + result = { + 'qobuz_track': { + 'name': qobuz_track.get('name', 'Unknown'), + 'artists': qobuz_track.get('artists', []), + }, + 'spotify_data': None, + 'match_data': None, + 'status': 'Error', + 'status_class': 'error', + 'spotify_track': '', + 'spotify_artist': '', + 'spotify_album': '', + 'error': str(e), + 'discovery_source': discovery_source, + 'index': i + } + state['discovery_results'].append(result) + state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) + + # Mark as complete + state['phase'] = 'discovered' + state['status'] = 'discovered' + state['discovery_progress'] = 100 + + source_label = discovery_source.upper() + deps.add_activity_item("", f"Qobuz Discovery Complete ({source_label})", f"'{playlist['name']}' - {successful_discoveries}/{len(tracks)} tracks found", "Now") + + logger.info(f"Qobuz discovery complete ({source_label}): {successful_discoveries}/{len(tracks)} tracks found") + + deps.sync_discovery_results_to_mirrored('qobuz', playlist_id, state.get('discovery_results', []), discovery_source, profile_id=state.get('_profile_id', 1)) + + except Exception as e: + logger.error(f"Error in Qobuz discovery worker: {e}") + if playlist_id in deps.qobuz_discovery_states: + deps.qobuz_discovery_states[playlist_id]['phase'] = 'error' + deps.qobuz_discovery_states[playlist_id]['status'] = f'error: {str(e)}' + finally: + deps.resume_enrichment_workers(_ew_state, 'Qobuz discovery') diff --git a/core/discovery/sync.py b/core/discovery/sync.py index 7d63a51f..a77b5d3e 100644 --- a/core/discovery/sync.py +++ b/core/discovery/sync.py @@ -397,9 +397,12 @@ def run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, p } logger.info(f"Sync finished for {playlist_id} - state updated") - # Set playlist poster image if available (Plex, Jellyfin, Emby) + # Set playlist poster image if available (Plex, Jellyfin, Emby). + # Don't log the URL itself — it may carry an auth token (Plex + # X-Plex-Token / Jellyfin X-Emby-Token / Subsonic auth) that we + # don't want persisted to app.log. _synced = getattr(result, 'synced_tracks', 0) - logger.info(f"[PLAYLIST IMAGE] image_url={playlist_image_url!r}, synced_tracks={_synced}") + logger.info(f"[PLAYLIST IMAGE] has_image={bool(playlist_image_url)}, synced_tracks={_synced}") if playlist_image_url and _synced > 0: try: active_server = deps.config_manager.get_active_media_server() diff --git a/core/download_orchestrator.py b/core/download_orchestrator.py index e8231e09..1984c0a6 100644 --- a/core/download_orchestrator.py +++ b/core/download_orchestrator.py @@ -104,6 +104,31 @@ class DownloadOrchestrator: deezer_dl.reconnect(deezer_arl) deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac') + # Reload Amazon quality preference (T2Tunes needs no reconnect — public proxy) + amazon = self.client('amazon') + if amazon: + quality = config_manager.get('amazon_download.quality', 'flac') + amazon._quality = quality + amazon._allow_fallback = config_manager.get('amazon_download.allow_fallback', True) + if hasattr(amazon, '_client') and amazon._client: + amazon._client.preferred_codec = quality + + # Let registry-backed plugins refresh any config they cache at + # construction time. This covers Prowlarr-backed torrent / usenet + # clients without rebuilding the registry and losing active downloads. + for name, client in self.registry.all_plugins(): + if not hasattr(client, 'reload_settings'): + continue + try: + client.reload_settings() + logger.info("%s client settings reloaded", self.registry.display_name(name)) + except Exception as exc: + logger.warning( + "%s client settings reload failed: %s", + self.registry.display_name(name), + exc, + ) + # Reload download path for all clients that cache it. # Soulseek owns the path config and is reloaded above; every # other source mirrors that path so files all land in one @@ -277,11 +302,25 @@ class DownloadOrchestrator: chain = ['soulseek'] return chain - async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]: + async def search(self, query: str, timeout: int = None, progress_callback=None, + exclude_sources=None) -> Tuple[List[TrackResult], List[AlbumResult]]: """Search for tracks using configured source(s). Single-source modes route directly; hybrid mode delegates to - ``engine.search_with_fallback`` which tries the chain in order.""" + ``engine.search_with_fallback`` which tries the chain in order. + + ``exclude_sources`` (optional) is an iterable of source names + the caller wants filtered out of the hybrid chain. Used by the + per-track download worker to skip torrent / usenet for album- + context batches — those sources are release-level and don't + score meaningfully on per-track titles; the album-bundle flow + on the master worker handles them separately when they're the + single active source.""" if self.mode != 'hybrid': + # Single-source mode is opt-in; honour the user's choice even + # if it's torrent/usenet on an album batch (the master worker + # routes those through the album-bundle flow before per-track + # tasks ever fire, so search() being called here would be a + # non-album / wishlist / basic-search use case). client = self._client(self.mode) if not client: logger.error(f"{self.registry.display_name(self.mode)} client not available (failed to initialize)") @@ -290,6 +329,19 @@ class DownloadOrchestrator: return await client.search(query, timeout, progress_callback) chain = self._resolve_source_chain() + if exclude_sources: + blocked = {s.lower() for s in exclude_sources if s} + filtered = [s for s in chain if s.lower() not in blocked] + if filtered != chain: + logger.info( + "Hybrid search: excluding %s for this query (chain %s -> %s)", + sorted(blocked & {s.lower() for s in chain}), + " → ".join(chain), " → ".join(filtered) if filtered else "(empty)", + ) + chain = filtered + if not chain: + logger.warning("Hybrid search exhausted: no eligible sources after exclusion filter") + return [], [] logger.info(f"Hybrid search ({' → '.join(chain)}): {query}") return await self.engine.search_with_fallback(query, chain, timeout, progress_callback) @@ -319,7 +371,7 @@ class DownloadOrchestrator: return None # 2. Filter and validate results - _streaming_sources = ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') + _streaming_sources = ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon') is_streaming = tracks[0].username in _streaming_sources if tracks else False if is_streaming and expected_track: diff --git a/core/download_plugins/album_bundle.py b/core/download_plugins/album_bundle.py new file mode 100644 index 00000000..6a0fb246 --- /dev/null +++ b/core/download_plugins/album_bundle.py @@ -0,0 +1,217 @@ +"""Shared helpers for the album-bundle download flow. + +The torrent and usenet download plugins both implement a +``download_album_to_staging`` method that searches Prowlarr for a +whole release, hands it to the active downloader, walks the +resulting audio files, and copies them into the staging folder. The +two implementations share the same release-picker heuristic and the +same staging-path collision logic. + +Pulled out of ``core/download_plugins/torrent.py`` so the usenet +plugin doesn't have to import private helpers from a sibling +plugin (Cin's "no leaky module boundaries" standard). + +Also exposes ``atomic_copy_to_staging`` — the audio file is copied +to a ``.tmp.`` sidecar first and atomically renamed onto its +final extension. The Auto-Import worker filters by audio extension +so the in-flight ``.tmp`` file is never picked up mid-copy, closing +the race between the album-bundle copy loop and Auto-Import's +folder scan. +""" + +from __future__ import annotations + +import shutil +import time +import uuid +from pathlib import Path +from typing import Iterable, Optional + +from config.settings import config_manager +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.album_bundle") + + +# Album-pick size floor / ceiling. Single-track torrents (~10 MB) +# are rejected when bigger candidates exist; anything past 3 GB is +# treated as suspicious (multi-disc box-set + scans + extras). +ALBUM_PICK_MIN_BYTES = 40 * 1024 * 1024 +ALBUM_PICK_MAX_BYTES = 3 * 1024 * 1024 * 1024 + + +# Quality-score weights for the album-pick heuristic. Mirrors the +# tier order in ``core/imports/file_ops.py``'s ``quality_tiers`` — +# higher number = preferred. +_QUALITY_SCORE = {'flac': 4, 'ogg': 3, 'aac': 2, 'mp3': 1} + + +# Default poll cadence + timeout for the album-download poll loop. +# Both are overridable through config so users with slow trackers +# / large box-sets can extend the deadline without editing code. +DEFAULT_POLL_INTERVAL_SECONDS = 2.0 +DEFAULT_POLL_TIMEOUT_SECONDS = 6 * 60 * 60 + + +def get_poll_interval() -> float: + """Return the per-poll sleep duration (seconds). Configurable via + ``download_source.album_bundle_poll_interval_seconds``.""" + raw = config_manager.get('download_source.album_bundle_poll_interval_seconds', + DEFAULT_POLL_INTERVAL_SECONDS) + try: + value = float(raw) + if value > 0: + return value + except (TypeError, ValueError): + pass + return DEFAULT_POLL_INTERVAL_SECONDS + + +def get_poll_timeout() -> float: + """Return the total deadline for an album-bundle download + (seconds). Configurable via + ``download_source.album_bundle_timeout_seconds``.""" + raw = config_manager.get('download_source.album_bundle_timeout_seconds', + DEFAULT_POLL_TIMEOUT_SECONDS) + try: + value = float(raw) + if value > 0: + return value + except (TypeError, ValueError): + pass + return DEFAULT_POLL_TIMEOUT_SECONDS + + +def quality_score(title: str, quality_guess) -> int: + """Map a release title's inferred quality to a sortable integer. + + ``quality_guess`` is the function from each plugin that maps a + title string to a quality string ('flac' / 'mp3' / etc.) — passed + in so this module doesn't have to import either plugin and risk + a circular import.""" + return _QUALITY_SCORE.get(quality_guess(title) or '', 0) + + +def pick_best_album_release(candidates, quality_guess) -> Optional[object]: + """Pick the single best torrent / NZB for an album-bundle download. + + Heuristic, in priority order: + 1. Reasonable album-ish size (40 MB – 3 GB) — drops single-track + releases that snuck in and quarantines suspicious giants. + 2. Higher seeders > lower (dead torrents = dead downloads). + Usenet releases use ``grabs`` as a popularity proxy when + seeders is None. + 3. Higher quality (FLAC > AAC > MP3) inferred from title. + 4. Larger size as tiebreaker (often = higher bitrate). + """ + if not candidates: + return None + sized = [c for c in candidates + if ALBUM_PICK_MIN_BYTES <= (c.size or 0) <= ALBUM_PICK_MAX_BYTES] + pool = sized or list(candidates) + if not pool: + return None + + def _score(c) -> tuple: + seeders = c.seeders if c.seeders is not None else (c.grabs or 0) + return (seeders, quality_score(c.title or '', quality_guess), c.size or 0) + + return max(pool, key=_score) + + +def unique_staging_path(staging_dir: Path, src: Path) -> Path: + """Return a destination path inside ``staging_dir`` that doesn't + collide with an existing file. Appends ``_1``, ``_2``, ... before + the extension when needed; gives up after 1000 candidates and + returns the unsuffixed path so the caller will overwrite (better + than infinite loop or crash).""" + dest = staging_dir / src.name + if not dest.exists(): + return dest + stem = dest.stem + suffix = dest.suffix + for i in range(1, 1000): + candidate = staging_dir / f"{stem}_{i}{suffix}" + if not candidate.exists(): + return candidate + return dest + + +def atomic_copy_to_staging(src: Path, dest: Path) -> bool: + """Copy ``src`` to ``dest`` without exposing a partial file to + folder scanners. + + The Auto-Import worker filters by audio extension when scanning + Staging — see ``AUDIO_EXTENSIONS`` in ``core/auto_import_worker.py``. + Naming the in-flight file ``.tmp.`` keeps it + invisible until the rename atomically swings it to its final + extension. ``os.replace`` (used by ``Path.rename`` on Python 3.x) + is atomic on the same filesystem, so Auto-Import either sees the + file at its final name (complete) or doesn't see it at all + (in flight). + + Returns True on success, False on copy / rename failure. Caller + is expected to log the failure case so we don't double-log here. + """ + tmp = dest.with_name(f"{dest.name}.tmp.{uuid.uuid4().hex[:8]}") + try: + shutil.copy2(src, tmp) + except Exception: + # Best-effort cleanup of the partial file. If unlink fails + # (locked, permissions) we leave it — Auto-Import ignores it + # anyway because of the .tmp extension. + try: + if tmp.exists(): + tmp.unlink() + except Exception as cleanup_exc: + logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc) + raise + try: + tmp.replace(dest) + return True + except Exception: + try: + tmp.unlink(missing_ok=True) + except Exception as cleanup_exc: + logger.debug("album_bundle tmp cleanup failed: %s", cleanup_exc) + raise + + +def copy_audio_files_atomically( + sources: Iterable[Path], staging_dir: Path, +) -> list: + """Convenience wrapper: pick a non-colliding staging path for + each source, copy via ``atomic_copy_to_staging``. Returns the + list of final destination paths (as strings). Files that fail + to copy are logged and skipped; the caller decides what to do + with a partial result.""" + staging_dir.mkdir(parents=True, exist_ok=True) + out: list = [] + for src in sources: + dest = unique_staging_path(staging_dir, src) + try: + atomic_copy_to_staging(src, dest) + out.append(str(dest)) + except Exception as e: + logger.warning("[album_bundle] Failed to stage %s -> %s: %s", src, dest, e) + return out + + +# Re-export so callers don't have to remember which module owns +# what. The ``time`` import is kept so plugins can ``from +# core.download_plugins.album_bundle import time`` if they want to, +# avoiding a second std-lib import line for a single use. +__all__ = [ + "ALBUM_PICK_MIN_BYTES", + "ALBUM_PICK_MAX_BYTES", + "DEFAULT_POLL_INTERVAL_SECONDS", + "DEFAULT_POLL_TIMEOUT_SECONDS", + "atomic_copy_to_staging", + "copy_audio_files_atomically", + "get_poll_interval", + "get_poll_timeout", + "pick_best_album_release", + "quality_score", + "time", + "unique_staging_path", +] diff --git a/core/download_plugins/registry.py b/core/download_plugins/registry.py index ed62e899..2ca5ebb8 100644 --- a/core/download_plugins/registry.py +++ b/core/download_plugins/registry.py @@ -38,7 +38,10 @@ from core.download_plugins.base import DownloadSourcePlugin # than the legacy module-top imports here. Importing everything at # registry-load time pins the bindings the same way the legacy # orchestrator did. +from core.amazon_download_client import AmazonDownloadClient from core.deezer_download_client import DeezerDownloadClient +from core.download_plugins.torrent import TorrentDownloadPlugin +from core.download_plugins.usenet import UsenetDownloadPlugin from core.hifi_client import HiFiClient from core.lidarr_download_client import LidarrDownloadClient from core.qobuz_client import QobuzClient @@ -176,6 +179,7 @@ def build_default_registry() -> DownloadPluginRegistry: """ registry = DownloadPluginRegistry() + registry.register(PluginSpec(name='amazon', factory=AmazonDownloadClient, display_name='Amazon Music')) registry.register(PluginSpec(name='soulseek', factory=SoulseekClient, display_name='Soulseek')) registry.register(PluginSpec(name='youtube', factory=YouTubeClient, display_name='YouTube')) registry.register(PluginSpec(name='tidal', factory=TidalDownloadClient, display_name='Tidal')) @@ -188,5 +192,7 @@ def build_default_registry() -> DownloadPluginRegistry: aliases=('deezer_dl',))) registry.register(PluginSpec(name='lidarr', factory=LidarrDownloadClient, display_name='Lidarr')) registry.register(PluginSpec(name='soundcloud',factory=SoundcloudClient, display_name='SoundCloud')) + registry.register(PluginSpec(name='torrent', factory=TorrentDownloadPlugin, display_name='Torrent (Prowlarr)')) + registry.register(PluginSpec(name='usenet', factory=UsenetDownloadPlugin, display_name='Usenet (Prowlarr)')) return registry diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py new file mode 100644 index 00000000..388cbe98 --- /dev/null +++ b/core/download_plugins/torrent.py @@ -0,0 +1,666 @@ +"""TorrentDownloadPlugin — composes Prowlarr search + torrent client +adapter + archive_pipeline into a uniform download source. + +Two flows: + +**Per-track flow** (basic search, single-track wishlist) — +1. ``search(query)`` calls ``ProwlarrClient.search`` filtered to + ``protocol='torrent'`` results, projects releases into + ``TrackResult`` / ``AlbumResult`` shaped objects the existing + search UI already understands. Encodes the indexer's + ``downloadUrl`` (or magnet URI) into the filename so + ``download()`` can recover it. +2. ``download(username, filename, ...)`` decodes the URL, asks the + active torrent adapter (qBittorrent, Transmission, or Deluge per + user's settings) to add it, spawns a background thread that + polls the adapter for completion. +3. On completion the thread walks the adapter-reported save path + via ``archive_pipeline.collect_audio_after_extraction`` and + exposes the full audio-file list. Post-processing can then pick + the requested track from a completed release instead of importing + the first file blindly. + +**Album-bundle flow** (album-context batch downloads — wired in +``core/downloads/master.py``) — +4. ``download_album_to_staging(album, artist, staging_dir)`` does + ONE Prowlarr search for the whole release, picks the best + torrent (prefers FLAC, decent seeders, reasonable size), + downloads it, extracts archives if needed, copies every audio + file into the staging directory. The existing per-track + ``try_staging_match`` flow then finds + imports each track by + fuzzy title match against the staged files. Per-track Prowlarr + queries never fire — track titles like "Luther (with SZA)" + would match album torrents like "GNX (2024) [FLAC]" at near- + zero confidence and break the per-track dispatch. + +Limitations: +- ``save_path`` is the torrent client's view of the disk. If + SoulSync runs on a different host than qBit / Trans / Deluge, + the post-processing pipeline can't see those files. The plugin + works fine for the all-on-one-box case (most users); remote + setups will need a future sync step (rclone / SMB / Docker + bind mount). +- Track-level metadata isn't available until after download. + Search results carry only the release title + indexer metadata; + individual track names are populated when the matching pipeline + walks the extracted audio files. +""" + +from __future__ import annotations + +import asyncio +import re +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from config.settings import config_manager +from core.archive_pipeline import collect_audio_after_extraction +from core.download_plugins.album_bundle import ( + copy_audio_files_atomically, + get_poll_interval, + get_poll_timeout, + pick_best_album_release, +) +from core.download_plugins.base import DownloadSourcePlugin +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult +from core.prowlarr_client import ( + DEFAULT_MUSIC_CATEGORIES, + ProwlarrClient, + ProwlarrSearchResult, +) +from core.torrent_clients import get_active_adapter as get_active_torrent_adapter +from utils.async_helpers import run_async +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.torrent") + + +# Separator used to encode the download URL inside the filename +# field. Same convention Lidarr / YouTube use for embedding their +# own opaque identifiers — ``||``. +_FILENAME_SEP = '||' + +# Adapter states that count as the download being on-disk and +# safe to walk. ``seeding`` and ``completed`` both mean the +# bits are there; the user can pause seeding manually if they +# don't want to keep sharing. +_COMPLETE_STATES = frozenset(['seeding', 'completed']) + +# Poll cadence / timeout — both pull from config via the shared +# album_bundle helpers so users can extend the deadline for slow +# trackers without editing source. Kept as module aliases so the +# per-track flow at the bottom of this file can still import them +# under the legacy names without re-reading config every loop. +_POLL_TIMEOUT_SECONDS = get_poll_timeout() +_POLL_INTERVAL_SECONDS = get_poll_interval() + + +class TorrentDownloadPlugin(DownloadSourcePlugin): + """Torrent download source backed by Prowlarr + an active + torrent client adapter.""" + + def __init__(self) -> None: + self._prowlarr = ProwlarrClient() + # Track every download we've kicked off. Keyed by our own + # uuid — NOT the adapter's hash — because the orchestrator + # owns the lifecycle and we need a stable id even before + # the adapter has assigned one. + self.active_downloads: Dict[str, Dict[str, Any]] = {} + self._lock = threading.Lock() + self.shutdown_check = None + + def set_shutdown_check(self, check_callable): + self.shutdown_check = check_callable + + def reload_settings(self) -> None: + self._prowlarr.reload_settings() + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def is_configured(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_torrent_adapter() + return bool(adapter and adapter.is_configured()) + + async def check_connection(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_torrent_adapter() + if not adapter or not adapter.is_configured(): + return False + # Probe both sides. A torrent download is useless if either + # the indexer or the downloader is unreachable. + prowlarr_ok = await self._prowlarr.check_connection() + if not prowlarr_ok: + return False + return await adapter.check_connection() + + # ------------------------------------------------------------------ + # Search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback=None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + if not self._prowlarr.is_configured(): + return ([], []) + try: + indexer_ids = _parse_indexer_id_filter() + results = await self._prowlarr.search( + query, + categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=indexer_ids, + ) + except Exception as e: + logger.error("Torrent plugin search failed: %s", e) + return ([], []) + return self._project_results(results) + + def _project_results( + self, results: List[ProwlarrSearchResult] + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + """Turn Prowlarr releases into TrackResult / AlbumResult + shaped objects. One TrackResult + one AlbumResult per + release — Prowlarr search hits are at the release level, + not the track level, so we can't synthesise track listings + without downloading the actual torrent.""" + tracks: List[TrackResult] = [] + albums: List[AlbumResult] = [] + for result in results: + if result.protocol != 'torrent': + continue + download_url = result.magnet_uri or result.download_url + if not download_url: + continue + filename = f"{download_url}{_FILENAME_SEP}{result.title}" + quality = _guess_quality_from_title(result.title) + parsed_artist, parsed_title = _parse_release_title(result.title) + tr = TrackResult( + username='torrent', + filename=filename, + size=result.size, + bitrate=None, + duration=None, + quality=quality, + # Torrent results don't have per-uploader slot / queue + # data the way Soulseek does. Fill with neutral values + # so the quality_score doesn't punish them artificially. + free_upload_slots=max(1, result.seeders or 0), + upload_speed=0, + queue_length=0, + # Pre-fill artist + title so TrackResult.__post_init__ + # doesn't auto-parse the filename — our filename starts + # with the indexer download URL, which would otherwise + # show up as "by download?apikey=..." in the UI. + artist=parsed_artist or result.indexer_name or 'Torrent', + title=parsed_title or result.title, + album=parsed_title or None, + track_number=None, + _source_metadata={ + 'indexer': result.indexer_name, + 'indexer_id': result.indexer_id, + 'seeders': result.seeders, + 'leechers': result.leechers, + 'grabs': result.grabs, + 'protocol': 'torrent', + }, + ) + tracks.append(tr) + albums.append(AlbumResult( + username='torrent', + album_path=f"torrent/{result.guid}", + album_title=parsed_title or result.title, + artist=parsed_artist or None, + track_count=1, # unknown until download finishes + total_size=result.size, + tracks=[tr], + dominant_quality=quality, + year=None, + )) + return tracks, albums + + # ------------------------------------------------------------------ + # Download + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + if not self.is_configured(): + return None + download_url, display_name = _decode_filename(filename) + if not download_url: + logger.error("Torrent download missing URL in filename: %r", filename) + return None + + download_id = str(uuid.uuid4()) + with self._lock: + self.active_downloads[download_id] = { + 'id': download_id, + 'filename': filename, + 'username': 'torrent', + 'display_name': display_name, + 'state': 'Initializing', + 'progress': 0.0, + 'size': file_size, + 'transferred': 0, + 'speed': 0, + 'file_path': None, + 'audio_files': [], + 'torrent_hash': None, + 'error': None, + } + + thread = threading.Thread( + target=self._download_thread, + args=(download_id, download_url, display_name), + daemon=True, + name=f'torrent-dl-{download_id[:8]}', + ) + thread.start() + return download_id + + def _download_thread(self, download_id: str, download_url: str, display_name: str) -> None: + """Background worker: hand the URL to the active adapter, + poll until done, then walk the resulting directory.""" + adapter = get_active_torrent_adapter() + if adapter is None or not adapter.is_configured(): + self._mark_error(download_id, "No torrent client configured") + return + + try: + torrent_hash = run_async(adapter.add_torrent(download_url)) + except Exception as e: + self._mark_error(download_id, f"add_torrent failed: {e}") + return + if not torrent_hash: + self._mark_error(download_id, "Torrent client refused the URL") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['torrent_hash'] = torrent_hash + row['state'] = 'InProgress, Downloading' + + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return + try: + status = run_async(adapter.get_status(torrent_hash)) + except Exception as e: + logger.warning("Torrent poll error for %s: %s", torrent_hash, e) + status = None + + if status is None: + # Adapter forgot about the torrent — probably user-removed. + self._mark_error(download_id, "Torrent disappeared from client") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['progress'] = status.progress * 100.0 + row['transferred'] = status.downloaded + row['speed'] = status.download_speed + row['size'] = status.size or row.get('size', 0) + row['state'] = _adapter_state_to_display(status.state) + row['error'] = status.error + if status.save_path: + last_save_path = status.save_path + + if status.state in _COMPLETE_STATES: + self._finalize_download(download_id, last_save_path) + return + if status.state == 'error': + self._mark_error(download_id, status.error or "Torrent client reported error") + return + + time.sleep(_POLL_INTERVAL_SECONDS) + + self._mark_error(download_id, "Torrent download timed out") + + def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None: + """Adapter said complete. Walk the directory + pick the + first audio file as the canonical ``file_path``.""" + if not save_path: + self._mark_error(download_id, "Torrent completed but no save_path reported") + return + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + self._mark_error(download_id, f"Post-extract walk failed: {e}") + return + if not audio_files: + self._mark_error(download_id, f"No audio files found in {save_path}") + return + primary = audio_files[0] + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Succeeded' + row['progress'] = 100.0 + row['file_path'] = str(primary) + row['audio_files'] = [str(path) for path in audio_files] + logger.info("Torrent download complete: %s -> %s (%d audio files)", + download_id[:8], primary.name, len(audio_files)) + + def _mark_error(self, download_id: str, message: str) -> None: + logger.error("Torrent download %s failed: %s", download_id[:8], message) + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Errored' + row['error'] = message + + # ------------------------------------------------------------------ + # Status / lifecycle + # ------------------------------------------------------------------ + + async def get_all_downloads(self) -> List[DownloadStatus]: + with self._lock: + rows = list(self.active_downloads.values()) + return [_row_to_status(r) for r in rows] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + with self._lock: + row = self.active_downloads.get(download_id) + if row is None: + return None + return _row_to_status(row) + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + adapter = get_active_torrent_adapter() + with self._lock: + row = self.active_downloads.get(download_id) + torrent_hash = row.get('torrent_hash') if row else None + if adapter and torrent_hash: + try: + await adapter.remove(torrent_hash, delete_files=remove) + except Exception as e: + logger.warning("Torrent cancel via adapter failed: %s", e) + with self._lock: + if remove: + self.active_downloads.pop(download_id, None) + else: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Cancelled' + return True + + async def clear_all_completed_downloads(self) -> bool: + with self._lock: + for did in list(self.active_downloads.keys()): + state = self.active_downloads[did].get('state', '') + if state.startswith('Completed') or state == 'Cancelled': + self.active_downloads.pop(did, None) + return True + + # ------------------------------------------------------------------ + # Album-bundle flow + # ------------------------------------------------------------------ + + def download_album_to_staging( + self, + album_name: str, + artist_name: str, + staging_dir: str, + progress_callback=None, + ) -> Dict[str, Any]: + """One-shot album download: search Prowlarr for the whole + release, pick the best torrent, fetch it, extract if needed, + copy every audio file into ``staging_dir`` so the existing + ``try_staging_match`` flow can hand each track off to the + post-processing pipeline. + + ``progress_callback`` is called with a dict on each state + change so the batch UI can show download progress without + waiting for the whole thing. + + Returns ``{'success': bool, 'files': [paths], 'error': str|None}``. + """ + result: Dict[str, Any] = {'success': False, 'files': [], 'error': None} + if not self.is_configured(): + result['error'] = 'Torrent source not configured' + return result + + adapter = get_active_torrent_adapter() + if adapter is None or not adapter.is_configured(): + result['error'] = 'No active torrent client' + return result + + def _emit(state: str, **extra) -> None: + if progress_callback: + payload = {'state': state, **extra} + try: + progress_callback(payload) + except Exception as cb_exc: + logger.debug("[Torrent album] progress callback failed: %s", cb_exc) + + # Phase 1: search Prowlarr for the album. + query = f"{artist_name} {album_name}".strip() + _emit('searching', query=query) + try: + search_results = run_async(self._prowlarr.search( + query, categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=_parse_indexer_id_filter(), + )) + except Exception as e: + result['error'] = f'Prowlarr search failed: {e}' + return result + + candidates = [r for r in search_results + if r.protocol == 'torrent' and (r.magnet_uri or r.download_url)] + if not candidates: + result['error'] = f'No torrent results found for "{query}"' + return result + + picked = pick_best_album_release(candidates, _guess_quality_from_title) + if picked is None: + result['error'] = 'No suitable torrent candidate after filtering' + return result + + download_url = picked.magnet_uri or picked.download_url + logger.info("[Torrent album] Picked '%s' (size=%.1fMB seeders=%s indexer=%s)", + picked.title, picked.size / 1_048_576, picked.seeders, picked.indexer_name) + _emit('queued', release=picked.title, size=picked.size, seeders=picked.seeders) + + # Phase 2: hand to adapter. + try: + torrent_id = run_async(adapter.add_torrent(download_url)) + except Exception as e: + result['error'] = f'Torrent client refused the release: {e}' + return result + if not torrent_id: + result['error'] = 'Torrent client refused the release' + return result + + # Phase 3: poll until complete. + _emit('downloading', release=picked.title) + save_path = self._poll_album_download(adapter, torrent_id, picked.title, _emit) + if save_path is None: + result['error'] = 'Torrent download failed or timed out' + return result + + # Phase 4: extract + walk + copy to staging. + _emit('staging', release=picked.title) + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + result['error'] = f'Failed to walk audio files: {e}' + return result + if not audio_files: + result['error'] = f'No audio files found in {save_path}' + return result + + copied = copy_audio_files_atomically(audio_files, Path(staging_dir)) + if not copied: + result['error'] = 'No audio files copied to staging' + return result + logger.info("[Torrent album] Staged %d audio files for '%s'", len(copied), album_name) + _emit('staged', count=len(copied)) + result['success'] = True + result['files'] = copied + return result + + def _poll_album_download(self, adapter, torrent_id, title, emit) -> Optional[str]: + """Poll the adapter until the torrent is complete. Returns + the save path or ``None`` on timeout / failure.""" + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return None + try: + status = run_async(adapter.get_status(torrent_id)) + except Exception as e: + logger.warning("[Torrent album] Poll error: %s", e) + status = None + if status is None: + logger.error("[Torrent album] '%s' disappeared from client", title) + return None + emit('downloading', progress=status.progress, downloaded=status.downloaded, + speed=status.download_speed) + if status.save_path: + last_save_path = status.save_path + if status.state in _COMPLETE_STATES: + return last_save_path + if status.state == 'error': + logger.error("[Torrent album] '%s' errored: %s", title, status.error) + return None + time.sleep(_POLL_INTERVAL_SECONDS) + logger.error("[Torrent album] '%s' timed out", title) + return None + + +# --------------------------------------------------------------------------- +# Module-level helpers (pure functions — easy to unit-test) +# --------------------------------------------------------------------------- + + +def _decode_filename(filename: str) -> Tuple[Optional[str], str]: + """Pull the encoded download URL out of the ``filename`` string. + Returns ``(url, display_name)``. ``url`` is None when the string + has no separator.""" + if not filename or _FILENAME_SEP not in filename: + return (None, filename or '') + url, display = filename.split(_FILENAME_SEP, 1) + return (url, display) + + +def _parse_release_title(title: str) -> Tuple[str, str]: + """Split a release title into ``(artist, title)`` using the + ``Artist - Title`` / ``Artist - Album`` convention almost every + indexer follows. Returns ``('', title)`` when no dash is found. + + Without this, ``TrackResult.__post_init__`` runs the bare + filename through ``parse_filename_metadata`` — and our filename + starts with the indexer's download URL, so the auto-parser + extracts garbage like ``download?apikey=...`` as the artist + and shows it in the search-result UI's "by" line. Pre-filling + the artist field short-circuits the auto-parse. + """ + if not title: + return ('', '') + # Strip common quality / format tags so the dash split doesn't + # eat them — "Artist - Album [FLAC] (2020)" → "Artist", "Album". + cleaned = re.sub(r'\s*[\[\(][^\]\)]*[\]\)]\s*$', '', title.strip()) + # Look for the FIRST " - " (or "-" surrounded by content). Some + # release titles have multiple dashes (subtitle dashes); the + # first split is the artist/work boundary. + parts = re.split(r'\s+-\s+|\s+-(?=\S)|(?<=\S)-\s+', cleaned, maxsplit=1) + if len(parts) == 2: + artist = parts[0].strip() + rest = parts[1].strip() + # Reject obvious non-artist prefixes (URLs, hashes, single + # punctuation) so we don't propagate garbage. + if artist and not re.match(r'^https?:|^[a-f0-9]{32,}$', artist): + return (artist, rest or cleaned) + return ('', cleaned) + + +def _guess_quality_from_title(title: str) -> str: + """Read the quality hint from a release title — most music + torrents put the encoding right in the name (FLAC, MP3 320, + etc.). Falls back to ``'mp3'`` so quality_score doesn't crash.""" + if not title: + return 'mp3' + lower = title.lower() + if 'flac' in lower: + return 'flac' + if re.search(r'\b24[\s-]?bit\b', lower) or 'hi-?res' in lower: + return 'flac' + if 'aac' in lower: + return 'aac' + if 'ogg' in lower: + return 'ogg' + return 'mp3' + + +def _parse_indexer_id_filter() -> List[int]: + """Read the comma-separated indexer-ID allowlist from config. + Empty list = search every enabled indexer.""" + raw = (config_manager.get('prowlarr.indexer_ids', '') or '').strip() + if not raw: + return [] + out: List[int] = [] + for chunk in raw.split(','): + chunk = chunk.strip() + if not chunk: + continue + try: + out.append(int(chunk)) + except ValueError: + continue + return out + + +def _adapter_state_to_display(state: str) -> str: + """Translate the adapter-uniform state strings into the + ``'InProgress, Downloading'`` / ``'Completed, Succeeded'`` + style the existing UI expects (matches Soulseek + Lidarr).""" + mapping = { + 'queued': 'Queued', + 'downloading': 'InProgress, Downloading', + 'stalled': 'InProgress, Stalled', + 'seeding': 'Completed, Succeeded', + 'completed': 'Completed, Succeeded', + 'paused': 'Paused', + 'error': 'Completed, Errored', + } + return mapping.get(state, state.title()) + + +def _row_to_status(row: Dict[str, Any]) -> DownloadStatus: + return DownloadStatus( + id=row['id'], + filename=row['filename'], + username=row['username'], + state=row.get('state', 'Unknown'), + progress=float(row.get('progress', 0.0)), + size=int(row.get('size', 0)), + transferred=int(row.get('transferred', 0)), + speed=int(row.get('speed', 0)), + time_remaining=None, + file_path=row.get('file_path'), + audio_files=row.get('audio_files') or None, + ) diff --git a/core/download_plugins/types.py b/core/download_plugins/types.py index 46e250c3..59d52913 100644 --- a/core/download_plugins/types.py +++ b/core/download_plugins/types.py @@ -197,3 +197,4 @@ class DownloadStatus: speed: int time_remaining: Optional[int] = None file_path: Optional[str] = None + audio_files: Optional[List[str]] = None diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py new file mode 100644 index 00000000..419bc380 --- /dev/null +++ b/core/download_plugins/usenet.py @@ -0,0 +1,461 @@ +"""UsenetDownloadPlugin — composes Prowlarr search + usenet client +adapter + archive_pipeline into a uniform download source. + +Mirrors ``TorrentDownloadPlugin`` in shape and lifecycle (see that +module's docstring for the full pipeline rationale). Differences: + +- Search filters Prowlarr results to ``protocol='usenet'``. +- ``add_nzb`` replaces ``add_torrent``; for NZBs we usually have + a direct HTTP URL the indexer exposes via Prowlarr. +- Usenet clients (SABnzbd, NZBGet) typically auto-extract during + post-processing, so ``archive_pipeline.collect_audio_after_extraction`` + usually has nothing to extract and just walks loose files. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from core.archive_pipeline import collect_audio_after_extraction +from core.download_plugins.album_bundle import ( + copy_audio_files_atomically, + pick_best_album_release, +) +from core.download_plugins.base import DownloadSourcePlugin +from core.download_plugins.torrent import ( + _adapter_state_to_display, + _decode_filename, + _guess_quality_from_title, + _parse_indexer_id_filter, + _parse_release_title, + _row_to_status, + _COMPLETE_STATES, + _FILENAME_SEP, + _POLL_INTERVAL_SECONDS, + _POLL_TIMEOUT_SECONDS, +) +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult +from core.prowlarr_client import ( + DEFAULT_MUSIC_CATEGORIES, + ProwlarrClient, + ProwlarrSearchResult, +) +from core.usenet_clients import get_active_adapter as get_active_usenet_adapter +from utils.async_helpers import run_async +from utils.logging_config import get_logger + +logger = get_logger("download_plugins.usenet") + + +class UsenetDownloadPlugin(DownloadSourcePlugin): + """Usenet download source backed by Prowlarr + an active usenet + client adapter (SABnzbd or NZBGet).""" + + def __init__(self) -> None: + self._prowlarr = ProwlarrClient() + self.active_downloads: Dict[str, Dict[str, Any]] = {} + self._lock = threading.Lock() + self.shutdown_check = None + + def set_shutdown_check(self, check_callable): + self.shutdown_check = check_callable + + def reload_settings(self) -> None: + self._prowlarr.reload_settings() + + def is_configured(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_usenet_adapter() + return bool(adapter and adapter.is_configured()) + + async def check_connection(self) -> bool: + if not self._prowlarr.is_configured(): + return False + adapter = get_active_usenet_adapter() + if not adapter or not adapter.is_configured(): + return False + if not await self._prowlarr.check_connection(): + return False + return await adapter.check_connection() + + # ------------------------------------------------------------------ + # Search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback=None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + if not self._prowlarr.is_configured(): + return ([], []) + try: + indexer_ids = _parse_indexer_id_filter() + results = await self._prowlarr.search( + query, + categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=indexer_ids, + ) + except Exception as e: + logger.error("Usenet plugin search failed: %s", e) + return ([], []) + return self._project_results(results) + + def _project_results( + self, results: List[ProwlarrSearchResult] + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + tracks: List[TrackResult] = [] + albums: List[AlbumResult] = [] + for result in results: + if result.protocol != 'usenet': + continue + if not result.download_url: + continue + filename = f"{result.download_url}{_FILENAME_SEP}{result.title}" + quality = _guess_quality_from_title(result.title) + parsed_artist, parsed_title = _parse_release_title(result.title) + tr = TrackResult( + username='usenet', + filename=filename, + size=result.size, + bitrate=None, + duration=None, + quality=quality, + # Usenet doesn't expose per-uploader concurrency the way + # Soulseek does; fill in neutral non-punishing values. + free_upload_slots=1, + upload_speed=0, + queue_length=0, + # Pre-fill artist + title so TrackResult.__post_init__ + # doesn't auto-parse the filename — same URL-in-filename + # gotcha as the torrent plugin. + artist=parsed_artist or result.indexer_name or 'Usenet', + title=parsed_title or result.title, + album=parsed_title or None, + track_number=None, + _source_metadata={ + 'indexer': result.indexer_name, + 'indexer_id': result.indexer_id, + 'grabs': result.grabs, + 'protocol': 'usenet', + }, + ) + tracks.append(tr) + albums.append(AlbumResult( + username='usenet', + album_path=f"usenet/{result.guid}", + album_title=parsed_title or result.title, + artist=parsed_artist or None, + track_count=1, + total_size=result.size, + tracks=[tr], + dominant_quality=quality, + year=None, + )) + return tracks, albums + + # ------------------------------------------------------------------ + # Download + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + if not self.is_configured(): + return None + nzb_url, display_name = _decode_filename(filename) + if not nzb_url: + logger.error("Usenet download missing URL in filename: %r", filename) + return None + + download_id = str(uuid.uuid4()) + with self._lock: + self.active_downloads[download_id] = { + 'id': download_id, + 'filename': filename, + 'username': 'usenet', + 'display_name': display_name, + 'state': 'Initializing', + 'progress': 0.0, + 'size': file_size, + 'transferred': 0, + 'speed': 0, + 'file_path': None, + 'audio_files': [], + 'job_id': None, + 'error': None, + } + + thread = threading.Thread( + target=self._download_thread, + args=(download_id, nzb_url), + daemon=True, + name=f'usenet-dl-{download_id[:8]}', + ) + thread.start() + return download_id + + def _download_thread(self, download_id: str, nzb_url: str) -> None: + adapter = get_active_usenet_adapter() + if adapter is None or not adapter.is_configured(): + self._mark_error(download_id, "No usenet client configured") + return + + try: + job_id = run_async(adapter.add_nzb(nzb_url)) + except Exception as e: + self._mark_error(download_id, f"add_nzb failed: {e}") + return + if not job_id: + self._mark_error(download_id, "Usenet client refused the NZB") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['job_id'] = job_id + row['state'] = 'InProgress, Downloading' + + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return + try: + status = run_async(adapter.get_status(job_id)) + except Exception as e: + logger.warning("Usenet poll error for %s: %s", job_id, e) + status = None + + if status is None: + self._mark_error(download_id, "Usenet job disappeared from client") + return + + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['progress'] = status.progress * 100.0 + row['transferred'] = status.downloaded + row['speed'] = status.download_speed + row['size'] = status.size or row.get('size', 0) + row['state'] = _adapter_state_to_display(status.state) + row['error'] = status.error + if status.save_path: + last_save_path = status.save_path + + if status.state in _COMPLETE_STATES: + self._finalize_download(download_id, last_save_path) + return + if status.state == 'failed': + self._mark_error(download_id, status.error or "Usenet client reported failure") + return + + time.sleep(_POLL_INTERVAL_SECONDS) + + self._mark_error(download_id, "Usenet download timed out") + + def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None: + if not save_path: + self._mark_error(download_id, "Usenet job completed but no save_path reported") + return + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + self._mark_error(download_id, f"Post-extract walk failed: {e}") + return + if not audio_files: + self._mark_error(download_id, f"No audio files found in {save_path}") + return + primary = audio_files[0] + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Succeeded' + row['progress'] = 100.0 + row['file_path'] = str(primary) + row['audio_files'] = [str(path) for path in audio_files] + logger.info("Usenet download complete: %s -> %s (%d audio files)", + download_id[:8], primary.name, len(audio_files)) + + def _mark_error(self, download_id: str, message: str) -> None: + logger.error("Usenet download %s failed: %s", download_id[:8], message) + with self._lock: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Completed, Errored' + row['error'] = message + + # ------------------------------------------------------------------ + # Status / lifecycle + # ------------------------------------------------------------------ + + async def get_all_downloads(self) -> List[DownloadStatus]: + with self._lock: + rows = list(self.active_downloads.values()) + return [_row_to_status(r) for r in rows] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + with self._lock: + row = self.active_downloads.get(download_id) + if row is None: + return None + return _row_to_status(row) + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + adapter = get_active_usenet_adapter() + with self._lock: + row = self.active_downloads.get(download_id) + job_id = row.get('job_id') if row else None + if adapter and job_id: + try: + await adapter.remove(job_id, delete_files=remove) + except Exception as e: + logger.warning("Usenet cancel via adapter failed: %s", e) + with self._lock: + if remove: + self.active_downloads.pop(download_id, None) + else: + row = self.active_downloads.get(download_id) + if row is not None: + row['state'] = 'Cancelled' + return True + + async def clear_all_completed_downloads(self) -> bool: + with self._lock: + for did in list(self.active_downloads.keys()): + state = self.active_downloads[did].get('state', '') + if state.startswith('Completed') or state == 'Cancelled': + self.active_downloads.pop(did, None) + return True + + # ------------------------------------------------------------------ + # Album-bundle flow + # ------------------------------------------------------------------ + + def download_album_to_staging( + self, + album_name: str, + artist_name: str, + staging_dir: str, + progress_callback=None, + ) -> Dict[str, Any]: + """Usenet sibling of ``TorrentDownloadPlugin.download_album_to_staging``. + See that method's docstring for the contract.""" + result: Dict[str, Any] = {'success': False, 'files': [], 'error': None} + if not self.is_configured(): + result['error'] = 'Usenet source not configured' + return result + + adapter = get_active_usenet_adapter() + if adapter is None or not adapter.is_configured(): + result['error'] = 'No active usenet client' + return result + + def _emit(state: str, **extra) -> None: + if progress_callback: + try: + progress_callback({'state': state, **extra}) + except Exception as cb_exc: + logger.debug("[Usenet album] progress callback failed: %s", cb_exc) + + query = f"{artist_name} {album_name}".strip() + _emit('searching', query=query) + try: + search_results = run_async(self._prowlarr.search( + query, categories=DEFAULT_MUSIC_CATEGORIES, + indexer_ids=_parse_indexer_id_filter(), + )) + except Exception as e: + result['error'] = f'Prowlarr search failed: {e}' + return result + + candidates = [r for r in search_results + if r.protocol == 'usenet' and r.download_url] + if not candidates: + result['error'] = f'No usenet results found for "{query}"' + return result + + picked = pick_best_album_release(candidates, _guess_quality_from_title) + if picked is None: + result['error'] = 'No suitable NZB candidate after filtering' + return result + + logger.info("[Usenet album] Picked '%s' (size=%.1fMB grabs=%s indexer=%s)", + picked.title, picked.size / 1_048_576, picked.grabs, picked.indexer_name) + _emit('queued', release=picked.title, size=picked.size, grabs=picked.grabs) + + try: + job_id = run_async(adapter.add_nzb(picked.download_url)) + except Exception as e: + result['error'] = f'Usenet client refused the NZB: {e}' + return result + if not job_id: + result['error'] = 'Usenet client refused the NZB' + return result + + _emit('downloading', release=picked.title) + save_path = self._poll_album_download(adapter, job_id, picked.title, _emit) + if save_path is None: + result['error'] = 'Usenet download failed or timed out' + return result + + _emit('staging', release=picked.title) + try: + audio_files = collect_audio_after_extraction(Path(save_path)) + except Exception as e: + result['error'] = f'Failed to walk audio files: {e}' + return result + if not audio_files: + result['error'] = f'No audio files found in {save_path}' + return result + + copied = copy_audio_files_atomically(audio_files, Path(staging_dir)) + if not copied: + result['error'] = 'No audio files copied to staging' + return result + logger.info("[Usenet album] Staged %d audio files for '%s'", len(copied), album_name) + _emit('staged', count=len(copied)) + result['success'] = True + result['files'] = copied + return result + + def _poll_album_download(self, adapter, job_id, title, emit) -> Optional[str]: + deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS + last_save_path: Optional[str] = None + while time.monotonic() < deadline: + if self.shutdown_check and self.shutdown_check(): + return None + try: + status = run_async(adapter.get_status(job_id)) + except Exception as e: + logger.warning("[Usenet album] Poll error: %s", e) + status = None + if status is None: + logger.error("[Usenet album] '%s' disappeared from client", title) + return None + emit('downloading', progress=status.progress, downloaded=status.downloaded, + speed=status.download_speed) + if status.save_path: + last_save_path = status.save_path + if status.state in _COMPLETE_STATES: + return last_save_path + if status.state == 'failed': + logger.error("[Usenet album] '%s' failed: %s", title, status.error) + return None + time.sleep(_POLL_INTERVAL_SECONDS) + logger.error("[Usenet album] '%s' timed out", title) + return None diff --git a/core/downloads/album_bundle_dispatch.py b/core/downloads/album_bundle_dispatch.py new file mode 100644 index 00000000..09ec1a46 --- /dev/null +++ b/core/downloads/album_bundle_dispatch.py @@ -0,0 +1,209 @@ +"""Album-bundle dispatch for torrent / usenet single-source downloads. + +Lifted from ``run_full_missing_tracks_process`` so the master +worker doesn't carry a 90-line inline branch and so the gate logic +can be unit-tested in isolation. + +The gate fires only when ALL conditions hold: + +- Batch is an album-context download (``is_album_download`` flag). +- Active download source is ``torrent``, ``usenet``, or ``soulseek``. + In hybrid mode the caller may pass the first configured source as a + source override; later hybrid sources stay per-track to preserve fallback. +- Both album-name and artist-name are populated in batch context. +- The resolved plugin exposes ``download_album_to_staging``. + +When the gate engages it runs the plugin synchronously (the master +worker is already on a thread-pool executor) and mirrors the +plugin's lifecycle payloads into the batch state so the Downloads +page can render meaningful progress before per-track tasks exist. + +Return semantics: ``True`` means the gate handled the batch — the +master worker should stop and not run per-track analysis. ``False`` +means the gate didn't engage (or engaged-and-fell-back) — caller +continues the normal per-track flow. + +The ``BatchStateAccess`` Protocol exists so this module doesn't +import ``download_batches`` from runtime_state directly. The +caller (master worker) injects accessors so this module stays +testable without touching live runtime state. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any, Callable, Optional, Protocol + +logger = logging.getLogger(__name__) + + +class BatchStateAccess(Protocol): + """Narrow shim around the batch-state dict ops the dispatch needs. + + Two methods to keep the surface small: + - ``update_fields(batch_id, fields)`` — atomic merge into the + batch dict under tasks_lock. + - ``mark_failed(batch_id, error)`` — convenience for the failure + path (sets phase + error + album_bundle_state in one shot). + """ + + def update_fields(self, batch_id: str, fields: dict) -> None: ... + + def mark_failed(self, batch_id: str, error: str) -> None: ... + + +# Fields the album-bundle progress callback may carry. Anything in +# this set gets mirrored onto the batch row as ``album_bundle_`` +# so the Downloads page can render it without coupling to the +# specific payload shape. +_MIRRORED_KEYS = ('progress', 'release', 'speed', 'downloaded', + 'size', 'seeders', 'grabs', 'count', 'failed') + + +def is_eligible( + *, + mode: str, + is_album: bool, + album_name: str, + artist_name: str, +) -> bool: + """Pure predicate: does this batch even qualify for the album + flow? Separate from the resolution+run step so tests can pin + the gate logic without standing up a plugin.""" + if not is_album: + return False + if (mode or '').lower() not in ('torrent', 'usenet', 'soulseek'): + return False + if not (album_name or '').strip(): + return False + if not (artist_name or '').strip(): + return False + return True + + +def try_dispatch( + *, + batch_id: str, + is_album: bool, + album_context: Optional[dict], + artist_context: Optional[dict], + config_get: Callable[..., Any], + plugin_resolver: Callable[[str], Optional[Any]], + state: BatchStateAccess, + source_override: Optional[str] = None, + plugin_kwargs: Optional[dict] = None, +) -> bool: + """Attempt the album-bundle flow. Returns ``True`` iff the + master worker should return early (gate engaged and completed + — success OR failure). ``False`` means fall through to the + normal per-track flow. + + ``config_get`` is a callable shaped like ``config_manager.get``; + ``plugin_resolver`` resolves a source-name string to an + initialised plugin instance (or None); ``state`` is the + BatchStateAccess shim. Injecting these keeps the module + dependency-light + unit-testable. + """ + mode = (source_override or config_get('download_source.mode', 'soulseek') or 'soulseek').lower() + album_name = (album_context or {}).get('name') or '' + artist_name = (artist_context or {}).get('name') or '' + + if not is_eligible(mode=mode, is_album=is_album, + album_name=album_name, artist_name=artist_name): + return False + + album_name = album_name.strip() + artist_name = artist_name.strip() + + plugin = None + try: + plugin = plugin_resolver(mode) + except Exception as exc: + logger.warning("[Album Bundle] Could not resolve %s plugin: %s", mode, exc) + + if plugin is None or not hasattr(plugin, 'download_album_to_staging'): + logger.warning( + "[Album Bundle] Gate matched but plugin / context unavailable " + "(mode=%s album=%r artist=%r plugin=%s) — falling back to per-track flow", + mode, album_name, artist_name, + type(plugin).__name__ if plugin else None, + ) + return False + + staging_root = config_get( + 'download_source.album_bundle_staging_path', + 'storage/album_bundle_staging', + ) or 'storage/album_bundle_staging' + staging_dir = str(Path(staging_root) / _safe_batch_dirname(batch_id)) + logger.info( + "[Album Bundle] Engaging %s album flow for '%s' by '%s' -> %s", + mode, album_name, artist_name, staging_dir, + ) + state.update_fields(batch_id, { + 'phase': 'album_downloading', + 'album_bundle_state': 'searching', + 'album_bundle_source': mode, + 'album_bundle_staging_path': staging_dir, + 'album_bundle_private_staging': True, + }) + + def _emit(payload): + """Mirror plugin lifecycle into batch state for UI rendering.""" + try: + fields = {'album_bundle_state': payload.get('state', '')} + for key in _MIRRORED_KEYS: + if key in payload: + fields[f'album_bundle_{key}'] = payload[key] + state.update_fields(batch_id, fields) + except Exception as exc: + logger.debug("[Album Bundle] emit failed: %s", exc) + + try: + outcome = plugin.download_album_to_staging( + album_name, artist_name, staging_dir, _emit, + **(plugin_kwargs or {}), + ) + except Exception as exc: + logger.exception("[Album Bundle] %s plugin raised: %s", mode, exc) + outcome = {'success': False, 'error': f'Plugin error: {exc}'} + + if not outcome.get('success'): + err = outcome.get('error', 'Album bundle download failed') + if outcome.get('fallback'): + logger.warning( + "[Album Bundle] %s flow could not commit for '%s': %s — falling back to per-track flow", + mode, album_name, err, + ) + state.update_fields(batch_id, { + 'phase': 'analysis', + 'album_bundle_state': 'fallback', + 'album_bundle_error': err, + }) + return False + logger.error("[Album Bundle] %s flow failed for '%s': %s", + mode, album_name, err) + state.mark_failed(batch_id, err) + return True + + logger.info( + "[Album Bundle] %s staged %d files for '%s' — handing off to per-track staging matcher", + mode, len(outcome.get('files', [])), album_name, + ) + state.update_fields(batch_id, { + 'phase': 'analysis', + 'album_bundle_state': 'staged', + 'album_bundle_partial': bool(outcome.get('partial')), + 'album_bundle_expected_count': outcome.get('expected_count'), + 'album_bundle_completed_count': outcome.get('completed_count', len(outcome.get('files', []))), + }) + # Engaged-and-succeeded: we DON'T early-return because the + # per-track flow needs to run to create + complete the per-track + # task rows. Those tasks will hit try_staging_match and pull the + # files we just staged. + return False + + +def _safe_batch_dirname(batch_id: str) -> str: + safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch')) + return safe or 'batch' diff --git a/core/downloads/cancel.py b/core/downloads/cancel.py index 9536eda4..b375f8fa 100644 --- a/core/downloads/cancel.py +++ b/core/downloads/cancel.py @@ -45,6 +45,9 @@ _TERMINAL_STATUSES = { def cancel_single_download(download_orchestrator, run_async: Callable, download_id: str, username: str) -> bool: """Cancel one specific slskd download (with `remove=True`).""" + logger.info( + f"[CancelTrigger:api.manual_cancel_single] download_id={download_id} username={username}" + ) return run_async(download_orchestrator.cancel_download(download_id, username, remove=True)) diff --git a/core/downloads/candidates.py b/core/downloads/candidates.py index d74cfa9d..52700ca9 100644 --- a/core/downloads/candidates.py +++ b/core/downloads/candidates.py @@ -63,8 +63,21 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, Attempts to download with fallback candidate logic (matches GUI's retry_parallel_download_with_fallback). Returns True if successful, False if all candidates fail. """ - # Sort candidates by confidence (best first) - candidates.sort(key=lambda r: r.confidence, reverse=True) + # Sort candidates by match confidence first, then peer quality. Upstream + # Soulseek validation already considers peer speed/slots/queue when scores + # are close; preserve that signal here instead of flattening ties back to + # arbitrary slskd response order. + candidates.sort( + key=lambda r: ( + getattr(r, 'confidence', 0) or 0, + getattr(r, 'quality_score', 0) or 0, + getattr(r, 'upload_speed', 0) or 0, + -(getattr(r, 'queue_length', 0) or 0), + getattr(r, 'free_upload_slots', 0) or 0, + getattr(r, 'size', 0) or 0, + ), + reverse=True, + ) with tasks_lock: task = download_tasks.get(task_id) @@ -235,7 +248,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, # 1. Try track_info (from frontend, has album track data) tn = track_info.get('track_number', 0) if isinstance(track_info, dict) else 0 - dn = track_info.get('disc_number', 1) if isinstance(track_info, dict) else 1 + dn = (track_info.get('disc_number') or 1) if isinstance(track_info, dict) else 1 if tn and tn > 0: enhanced_payload['track_number'] = tn enhanced_payload['disc_number'] = dn @@ -255,7 +268,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, detailed_track = deps.spotify_client.get_track_details(track.id) if detailed_track and detailed_track.get('track_number'): enhanced_payload['track_number'] = detailed_track['track_number'] - enhanced_payload['disc_number'] = detailed_track.get('disc_number', 1) + enhanced_payload['disc_number'] = detailed_track.get('disc_number') or 1 got_track_number = True logger.info(f"[Context] Added track_number from API: {detailed_track['track_number']}, disc_number: {enhanced_payload['disc_number']}") @@ -330,6 +343,10 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, logger.warning(f"[Modal Worker] Task {task_id} cancelled after download {download_id} started - attempting to cancel download") # Try to cancel the download immediately try: + logger.info( + f"[CancelTrigger:candidates.worker_cancelled_during_download] " + f"download_id={download_id} username={username} task_id={task_id}" + ) deps.run_async(deps.download_orchestrator.cancel_download(download_id, username, remove=True)) logger.warning(f"Successfully cancelled active download {download_id}") except Exception as cancel_error: diff --git a/core/downloads/lifecycle.py b/core/downloads/lifecycle.py index 43e1e970..5891e8be 100644 --- a/core/downloads/lifecycle.py +++ b/core/downloads/lifecycle.py @@ -26,9 +26,11 @@ Lifted verbatim from web_server.py. Dependencies injected via from __future__ import annotations import logging +import shutil import time import traceback from dataclasses import dataclass +from pathlib import Path from typing import Any, Callable, Optional from core.downloads.history import record_sync_history_completion @@ -42,6 +44,47 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +def _safe_batch_dirname(batch_id: str) -> str: + safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch')) + return safe or 'batch' + + +def _cleanup_private_album_bundle_staging(batch_id: str, batch: dict) -> None: + """Best-effort cleanup for torrent/usenet private staging copies. + + The torrent/usenet clients keep their own completed download folders. + This only removes SoulSync's per-batch copy under album-bundle staging. + """ + if not batch.get('album_bundle_private_staging'): + return + if (batch.get('album_bundle_source') or '').lower() not in ('torrent', 'usenet'): + return + + staging_path = batch.get('album_bundle_staging_path') + if not staging_path: + return + + path = Path(staging_path) + expected_name = _safe_batch_dirname(batch_id) + if path.name != expected_name: + logger.warning( + "[Album Bundle] Refusing to clean private staging path with unexpected name: %s", + staging_path, + ) + return + if not path.exists(): + return + if not path.is_dir(): + logger.warning("[Album Bundle] Refusing to clean non-directory staging path: %s", staging_path) + return + + try: + shutil.rmtree(path) + logger.info("[Album Bundle] Cleaned private staging folder for batch %s: %s", batch_id, staging_path) + except Exception as exc: + logger.warning("[Album Bundle] Could not clean private staging folder %s: %s", staging_path, exc) + + @dataclass class LifecycleDeps: """Bundle of cross-cutting deps the batch lifecycle needs.""" @@ -173,7 +216,7 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life # Guard against double-calling: track which tasks have already been completed # This prevents active_count from being decremented multiple times for the same task - # (e.g. monitor detects completion AND post-processing calls this again) + # (e.g. status polling and post-processing both observe the same terminal task) # NOTE: On duplicate calls, we skip decrement/tracking but STILL check batch completion. # This is critical because the first call may see the task in 'post_processing' (not finished), # and the second call (from post-processing worker) arrives after the task is truly 'completed'. @@ -398,6 +441,7 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life logger.info(f"[Batch Manager] Batch {batch_id} complete - stopping monitor") deps.download_monitor.stop_monitoring(batch_id) + _cleanup_private_album_bundle_staging(batch_id, batch) # M3U REGENERATION: Regenerate M3U with real library paths now that # all post-processing (tagging, moving, DB writes) is complete. @@ -606,6 +650,7 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo logger.info(f"[Completion Check V2] Batch {batch_id} complete - stopping monitor") deps.download_monitor.stop_monitoring(batch_id) + _cleanup_private_album_bundle_staging(batch_id, batch) # REPAIR: Scan all album folders from this batch for track number issues if deps.repair_worker: diff --git a/core/downloads/master.py b/core/downloads/master.py index 4620a467..83842f17 100644 --- a/core/downloads/master.py +++ b/core/downloads/master.py @@ -31,13 +31,244 @@ import re import time import uuid from dataclasses import dataclass +from difflib import SequenceMatcher +from pathlib import Path from typing import Any, Callable +from core.downloads import album_bundle_dispatch as _album_bundle_dispatch from core.runtime_state import download_batches, download_tasks, tasks_lock logger = logging.getLogger(__name__) +_ALBUM_PREFLIGHT_MIN_SCORE = 0.62 +_EDITION_WORDS = { + 'deluxe', 'expanded', 'anniversary', 'special', 'platinum', 'bonus', + 'remaster', 'remastered', 'edition', 'version', +} +_VARIANT_WORDS = { + 'remix', 'rmx', 'acapella', 'a cappella', 'instrumental', 'karaoke', + 'live', 'demo', 'extended', +} +_ALBUM_BUNDLE_SOURCES = frozenset(('torrent', 'usenet', 'soulseek')) + + +def _norm_text(value: Any) -> str: + text = str(value or '').lower() + text = re.sub(r'[_./\\|()[\]{}:;,+]', ' ', text) + text = re.sub(r'[^a-z0-9\s-]', '', text) + text = re.sub(r'\s+', ' ', text).strip() + return text + + +def _similarity(left: Any, right: Any) -> float: + a = _norm_text(left) + b = _norm_text(right) + if not a or not b: + return 0.0 + if a == b: + return 1.0 + if a in b or b in a: + return min(len(a), len(b)) / max(len(a), len(b)) + return SequenceMatcher(None, a, b).ratio() + + +def _track_title_from_candidate(candidate: Any) -> str: + title = getattr(candidate, 'title', None) + if title: + return str(title) + filename = getattr(candidate, 'filename', '') or '' + stem = Path(filename.replace('\\', '/')).stem + stem = re.sub(r'^\s*(?:disc\s*)?\d+[-_.\s]+', '', stem, flags=re.IGNORECASE) + return stem + + +def _track_number_from_track(track_data: dict) -> int: + value = track_data.get('track_number') or track_data.get('trackNumber') or 0 + try: + return int(str(value).split('/')[0]) + except (TypeError, ValueError): + return 0 + + +def _track_number_from_candidate(candidate: Any) -> int: + value = getattr(candidate, 'track_number', None) or 0 + try: + return int(str(value).split('/')[0]) + except (TypeError, ValueError): + return 0 + + +def _folder_variant_penalty(expected_album_name: str, folder_text: str) -> float: + expected = _norm_text(expected_album_name) + folder = _norm_text(folder_text) + if not folder: + return 0.0 + + penalty = 0.0 + for word in _VARIANT_WORDS: + if word in folder and word not in expected: + penalty += 0.12 + for word in _EDITION_WORDS: + if word in folder and word not in expected: + penalty += 0.06 + return min(penalty, 0.30) + + +def _source_quality_score(source: Any) -> float: + score = getattr(source, 'quality_score', None) + if callable(score): + try: + return float(score()) + except Exception: + return 0.0 + try: + return float(score or 0.0) + except (TypeError, ValueError): + return 0.0 + + +def _album_context_richness(album_ctx: dict) -> int: + if not isinstance(album_ctx, dict): + return 0 + fields = ('id', 'name', 'release_date', 'total_tracks', 'album_type') + score = sum(1 for field in fields if album_ctx.get(field)) + images = album_ctx.get('images') + if images: + score += 1 + artists = album_ctx.get('artists') + if artists: + score += 1 + return score + + +def _score_album_folder(album_result: Any, album_context: dict, artist_context: dict, + tracks_json: list[dict], filtered_track_count: int) -> float: + """Score one slskd folder as a whole release, not as isolated tracks.""" + expected_album = str((album_context or {}).get('name') or '') + expected_artist = str((artist_context or {}).get('name') or '') + expected_count = int((album_context or {}).get('total_tracks') or len(tracks_json) or 0) + expected_year = str((album_context or {}).get('release_date') or '')[:4] + + folder_text = ' '.join( + str(getattr(album_result, attr, '') or '') + for attr in ('album_title', 'album_path') + ) + album_score = max( + _similarity(expected_album, getattr(album_result, 'album_title', '')), + _similarity(expected_album, getattr(album_result, 'album_path', '')), + ) + artist_score = max( + _similarity(expected_artist, getattr(album_result, 'artist', '')), + _similarity(expected_artist, getattr(album_result, 'album_path', '')), + ) + + actual_count = int(getattr(album_result, 'track_count', 0) or len(getattr(album_result, 'tracks', []) or [])) + if expected_count > 0 and actual_count > 0: + diff = abs(actual_count - expected_count) + if diff == 0: + count_score = 1.0 + elif diff <= 2: + count_score = 0.75 + elif diff <= 5: + count_score = 0.35 + else: + count_score = 0.0 + else: + count_score = 0.4 + + candidate_tracks = list(getattr(album_result, 'tracks', []) or []) + matched = 0 + expected_tracks = [ + (track_data, _norm_text(track_data.get('name', ''))) + for track_data in tracks_json + if track_data.get('name') + ] + for track_data, expected_title in expected_tracks: + expected_number = _track_number_from_track(track_data) + best = 0.0 + for candidate in candidate_tracks: + cand_title = _norm_text(_track_title_from_candidate(candidate)) + title_sim = _similarity(expected_title, cand_title) + cand_number = _track_number_from_candidate(candidate) + if expected_number and cand_number and expected_number == cand_number: + title_sim = min(1.0, title_sim + 0.12) + best = max(best, title_sim) + if best >= 0.72: + matched += 1 + coverage_score = matched / max(1, len(expected_tracks)) + + year_score = 0.5 + folder_year = str(getattr(album_result, 'year', '') or '') + if expected_year and folder_year: + year_score = 1.0 if expected_year == folder_year else 0.2 + elif expected_year and expected_year in _norm_text(folder_text): + year_score = 1.0 + + quality_count_score = min(1.0, filtered_track_count / max(1, expected_count or actual_count or 1)) + peer_score = _source_quality_score(album_result) + penalty = _folder_variant_penalty(expected_album, folder_text) + + score = ( + album_score * 0.24 + + artist_score * 0.16 + + count_score * 0.16 + + coverage_score * 0.28 + + year_score * 0.06 + + quality_count_score * 0.06 + + peer_score * 0.04 + - penalty + ) + return max(0.0, min(score, 1.0)) + + +def _resolve_soulseek_client(download_orchestrator: Any) -> Any: + if hasattr(download_orchestrator, 'client'): + try: + client = download_orchestrator.client('soulseek') + if client: + return client + except Exception as exc: + logger.debug("Soulseek client lookup through orchestrator failed: %s", exc) + return getattr(download_orchestrator, 'soulseek', download_orchestrator) + + +def _soulseek_album_preflight_enabled(config_manager: Any) -> bool: + mode = config_manager.get('download_source.mode', 'hybrid') + if mode == 'soulseek': + return True + if mode != 'hybrid': + return False + order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + if order: + return order[0] == 'soulseek' + primary = config_manager.get('download_source.hybrid_primary', '') + return primary == 'soulseek' + + +def _resolve_album_bundle_source(config_manager: Any) -> str: + """Return the album-bundle source for this batch. + + In single-source mode, the active source may own the whole album if + it supports album bundles. In hybrid mode, only the first source in + the configured order may claim the whole album; later sources remain + per-track fallback. + """ + mode = (config_manager.get('download_source.mode', 'soulseek') or 'soulseek').lower() + if mode in _ALBUM_BUNDLE_SOURCES: + return mode + if mode != 'hybrid': + return '' + + order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + first = '' + if order: + first = str(order[0] or '').lower() + else: + first = str(config_manager.get('download_source.hybrid_primary', '') or '').lower() + return first if first in _ALBUM_BUNDLE_SOURCES else '' + + @dataclass class MasterDeps: """Bundle of cross-cutting deps the master worker needs.""" @@ -64,6 +295,26 @@ class MasterDeps: reset_wishlist_auto_processing: Callable[[], None] +class _BatchStateAccessImpl: + """Concrete ``BatchStateAccess`` for the runtime ``download_batches`` + dict — wraps the lock + the existing-batch check so the album- + bundle dispatcher stays decoupled from runtime_state.""" + + def update_fields(self, batch_id: str, fields: dict) -> None: + with tasks_lock: + row = download_batches.get(batch_id) + if row is not None: + row.update(fields) + + def mark_failed(self, batch_id: str, error: str) -> None: + with tasks_lock: + row = download_batches.get(batch_id) + if row is not None: + row['phase'] = 'failed' + row['error'] = error + row['album_bundle_state'] = 'failed' + + def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: MasterDeps): """ A master worker that handles the entire missing tracks process: @@ -79,25 +330,52 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma download_batches[batch_id]['analysis_processed'] = 0 from database.music_database import MusicDatabase + from core.library import manual_library_match as _mlm db = MusicDatabase() active_server = deps.config_manager.get_active_media_server() analysis_results = [] # Get force download flag and album context from batch force_download_all = False + ignore_manual_matches = False batch_album_context = None batch_artist_context = None batch_is_album = False + batch_profile_id = 1 + batch_source = 'spotify' with tasks_lock: if batch_id in download_batches: force_download_all = download_batches[batch_id].get('force_download_all', False) + ignore_manual_matches = download_batches[batch_id].get('ignore_manual_matches', False) batch_is_album = download_batches[batch_id].get('is_album_download', False) batch_album_context = download_batches[batch_id].get('album_context') batch_artist_context = download_batches[batch_id].get('artist_context') + batch_profile_id = download_batches[batch_id].get('profile_id', 1) or 1 + batch_source = download_batches[batch_id].get('batch_source', 'spotify') or 'spotify' if force_download_all: logger.warning(f"[Force Download] Force download mode enabled for batch {batch_id} - treating all tracks as missing") + # Album-bundle gate for torrent / usenet single-source mode. + # See ``core/downloads/album_bundle_dispatch`` for the full + # narrow-gate rationale. Returns True iff the master worker + # should stop (gate fired and failed); False = engaged-and- + # succeeded OR didn't engage, both fall through to per-track. + _bundle_state = _BatchStateAccessImpl() + _album_bundle_source = _resolve_album_bundle_source(deps.config_manager) + if _album_bundle_source and _album_bundle_source != 'soulseek': + if _album_bundle_dispatch.try_dispatch( + batch_id=batch_id, + is_album=batch_is_album, + album_context=batch_album_context, + artist_context=batch_artist_context, + config_get=deps.config_manager.get, + plugin_resolver=deps.download_orchestrator.client, + state=_bundle_state, + source_override=_album_bundle_source, + ): + return + # Allow duplicate tracks across albums — when enabled, only skip tracks already # owned in THIS album, not tracks owned in other albums allow_duplicates = deps.config_manager.get('wishlist.allow_duplicate_tracks', True) @@ -167,6 +445,26 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma artists = track_data.get('artists', []) found, confidence = False, 0.0 + # Manual library matches are authoritative unless the user explicitly + # requested a force re-download from the normal download modal. + _stid = track_data.get('spotify_track_id') or track_data.get('source_track_id') or track_data.get('id', '') + if not ignore_manual_matches and _stid and _mlm.get_match_for_track( + db, batch_profile_id, track_data, default_source=batch_source + ): + logger.info(f"[Manual Match] '{track_name}' already matched in library — skipping download") + try: + deps.check_and_remove_track_from_wishlist_by_metadata(track_data) + except Exception as _wl_err: + logger.debug(f"[Manual Match] Wishlist removal attempt failed: {_wl_err}") + analysis_results.append({ + 'track_index': track_index, + 'track': track_data, + 'found': True, + 'confidence': 1.0, + 'match_reason': 'manual_library_match', + }) + continue + # Skip database check if force download is enabled if force_download_all: logger.warning(f"[Force Download] Skipping database check for '{track_name}' - treating as missing") @@ -174,14 +472,33 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma elif album_tracks_map: # Album-scoped matching: check against known album tracks first track_name_lower = track_name.lower().strip() - # Direct title match + # Issue #589 — strip suffixes that just repeat the album + # context (e.g. "Shy Away (MTV Unplugged Live)" on a + # "MTV Unplugged" album → "Shy Away") so album-owned + # tracks don't false-miss when the local DB stored the + # base title. Only fires inside the album-confirmed + # scope; global matching elsewhere is unchanged. + from core.matching.album_context_title import strip_redundant_album_suffix + _album_name_for_strip = (batch_album_context or {}).get('name', '') + _normalized_source_title = strip_redundant_album_suffix( + track_name, _album_name_for_strip + ).lower().strip() + # Direct title match (try both raw and normalized) if track_name_lower in album_tracks_map: found, confidence = True, 1.0 + elif _normalized_source_title and _normalized_source_title in album_tracks_map: + found, confidence = True, 1.0 else: - # Fuzzy match against album tracks using string similarity + # Fuzzy match against album tracks using string similarity. + # Compare BOTH the raw and normalized source titles — + # whichever scores higher wins. Preserves strict + # matching when the album doesn't imply version + # context (helper returns the input unchanged). best_sim = 0.0 for db_title_lower, _db_track in album_tracks_map.items(): - sim = db._string_similarity(track_name_lower, db_title_lower) + sim_raw = db._string_similarity(track_name_lower, db_title_lower) + sim_norm = db._string_similarity(_normalized_source_title, db_title_lower) if _normalized_source_title else 0.0 + sim = max(sim_raw, sim_norm) if sim > best_sim: best_sim = sim if best_sim >= 0.7: @@ -350,6 +667,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma batch_album_context = batch.get('album_context') batch_artist_context = batch.get('artist_context') batch_is_album = batch.get('is_album_download', False) + batch_private_album_bundle = bool(batch.get('album_bundle_private_staging')) batch_playlist_folder_mode = batch.get('playlist_folder_mode', False) batch_playlist_name = batch.get('playlist_name', 'Unknown Playlist') @@ -357,13 +675,9 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma # Only run pre-flight when Soulseek is the download source (or hybrid with soulseek) preflight_source = None preflight_tracks = None - dl_source_mode = deps.config_manager.get('download_source.mode', 'hybrid') - _dl_hybrid_order = deps.config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) - _dl_hybrid_first = _dl_hybrid_order[0] if _dl_hybrid_order else deps.config_manager.get('download_source.hybrid_primary', 'hifi') - soulseek_is_source = dl_source_mode == 'soulseek' or ( - dl_source_mode == 'hybrid' and _dl_hybrid_first == 'soulseek' - ) - if batch_is_album and batch_album_context and batch_artist_context and soulseek_is_source: + soulseek_is_source = _soulseek_album_preflight_enabled(deps.config_manager) + if (batch_is_album and batch_album_context and batch_artist_context + and soulseek_is_source and not batch_private_album_bundle): artist_name = batch_artist_context.get('name', '') album_name = batch_album_context.get('name', '') if artist_name and album_name: @@ -372,7 +686,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma _sr.info(f"[Album Pre-flight] Searching for '{artist_name} {album_name}'") logger.info(f"[Album Pre-flight] Searching Soulseek for complete album: '{artist_name} - {album_name}'") - slsk = deps.download_orchestrator.client('soulseek') if hasattr(deps.download_orchestrator, 'client') else deps.download_orchestrator + slsk = _resolve_soulseek_client(deps.download_orchestrator) # Try multiple query variations (banned keywords in artist/album name can return 0 results) album_queries = [f"{artist_name} {album_name}"] @@ -386,29 +700,57 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma album_results = [] track_results = [] + album_results_by_source = {} for aq in album_queries: _sr.info(f"[Album Pre-flight] Trying query: '{aq}'") track_results, album_results = deps.run_async(slsk.search(aq, timeout=30)) if album_results: _sr.info(f"[Album Pre-flight] Found {len(album_results)} album results with query: '{aq}'") - break - _sr.info(f"[Album Pre-flight] No album results for query: '{aq}'") + for ar in album_results: + key = (getattr(ar, 'username', ''), getattr(ar, 'album_path', '')) + if key[0] and key[1] and key not in album_results_by_source: + album_results_by_source[key] = ar + else: + _sr.info(f"[Album Pre-flight] No album results for query: '{aq}'") + album_results = list(album_results_by_source.values()) if album_results: - # Filter by quality preference - quality_filtered = [] + # Score complete folders as releases before falling back to per-track search. + scored_albums = [] for ar in album_results: filtered_tracks = slsk.filter_results_by_quality_preference(ar.tracks) if filtered_tracks: - quality_filtered.append((ar, len(filtered_tracks))) + folder_score = _score_album_folder( + ar, + batch_album_context, + batch_artist_context, + tracks_json, + len(filtered_tracks), + ) + scored_albums.append((ar, len(filtered_tracks), folder_score)) + _sr.info( + f"[Album Pre-flight] Candidate {ar.username}:{ar.album_path} " + f"score={folder_score:.3f}, tracks={ar.track_count}, " + f"quality_tracks={len(filtered_tracks)}" + ) - if quality_filtered: - # Sort by track count (most complete album first), then quality score - quality_filtered.sort(key=lambda x: (x[1], x[0].quality_score), reverse=True) - best_album = quality_filtered[0][0] + best_album = None + best_score = 0.0 + if scored_albums: + scored_albums.sort(key=lambda x: (x[2], x[1], x[0].quality_score), reverse=True) + best_album, _best_filtered_count, best_score = scored_albums[0] + if best_score < _ALBUM_PREFLIGHT_MIN_SCORE: + _sr.info( + f"[Album Pre-flight] Best folder score {best_score:.3f} below " + f"threshold {_ALBUM_PREFLIGHT_MIN_SCORE:.2f}; falling back" + ) + logger.warning("[Album Pre-flight] No Soulseek folder passed album-level validation") + best_album = None + + if best_album: _sr.info(f"[Album Pre-flight] Best album result: {best_album.username}:{best_album.album_path} " - f"({best_album.track_count} tracks, quality={best_album.dominant_quality})") + f"({best_album.track_count} tracks, quality={best_album.dominant_quality}, score={best_score:.3f})") logger.info(f"[Album Pre-flight] Found album folder: {best_album.username} — " f"{best_album.track_count} tracks ({best_album.dominant_quality})") @@ -437,7 +779,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma } preflight_tracks = best_album.tracks logger.info(f"[Album Pre-flight] Using {len(best_album.tracks)} tracks from search results (browse unavailable)") - else: + elif not scored_albums: _sr.info("[Album Pre-flight] No album results passed quality filter") logger.warning("[Album Pre-flight] No album results matched quality preferences") else: @@ -448,13 +790,44 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma logger.error(f"[Album Pre-flight] Search failed (non-fatal, falling back to track-by-track): {preflight_err}") deps.source_reuse_logger.info(f"[Album Pre-flight] Exception: {preflight_err}") + # Soulseek album bundles run after analysis so an already-owned + # album does not get downloaded just because the source supports a + # whole-folder flow. When preflight selected a folder, pass that + # exact source into the bundle downloader so we keep the richer + # tracklist-aware scoring instead of doing a weaker second pick. + _bundle_state = _BatchStateAccessImpl() + _album_bundle_source = _resolve_album_bundle_source(deps.config_manager) + if _album_bundle_source == 'soulseek': + if _album_bundle_dispatch.try_dispatch( + batch_id=batch_id, + is_album=batch_is_album, + album_context=batch_album_context, + artist_context=batch_artist_context, + config_get=deps.config_manager.get, + plugin_resolver=deps.download_orchestrator.client, + state=_bundle_state, + source_override=_album_bundle_source, + plugin_kwargs={ + 'preferred_source': preflight_source, + 'preferred_tracks': preflight_tracks, + } if preflight_source and preflight_tracks else None, + ): + return + with tasks_lock: if batch_id not in download_batches: return download_batches[batch_id]['phase'] = 'downloading' # Store album pre-flight results on batch for source reuse - if preflight_source and preflight_tracks: + # unless the Soulseek album-bundle path already staged a private + # release. Task workers check source reuse before staging match, so + # preloading here would make the staged happy path re-download. + if ( + preflight_source + and preflight_tracks + and not download_batches[batch_id].get('album_bundle_private_staging') + ): download_batches[batch_id]['last_good_source'] = preflight_source download_batches[batch_id]['source_folder_tracks'] = preflight_tracks download_batches[batch_id]['failed_sources'] = set() @@ -464,7 +837,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma # Use ALL tracks (tracks_json), not just missing ones, to correctly detect multi-disc # even when only one disc has missing tracks if batch_is_album and batch_album_context: - total_discs = max((t.get('disc_number', 1) for t in tracks_json), default=1) + total_discs = max((t.get('disc_number') or 1 for t in tracks_json), default=1) batch_album_context['total_discs'] = total_discs if total_discs > 1: logger.info(f"[Multi-Disc] Detected {total_discs} discs for album '{batch_album_context.get('name')}'") @@ -473,6 +846,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma # Wishlist tracks aren't batch_is_album but each track has disc_number in spotify_data wishlist_album_disc_counts = {} wishlist_album_artist_map = {} # album_id -> resolved artist context (consistent per album) + wishlist_album_context_map = {} # album_id -> richest shared album context if playlist_id == 'wishlist': import json as _json # First pass: collect disc_number and resolve ONE artist per album @@ -488,11 +862,15 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma # Fallback album key: use album name when ID is missing (e.g. mirrored playlist tracks) if not album_id and isinstance(album_val, dict) and album_val.get('name'): album_id = f"_name_{album_val['name'].lower().strip()}" - disc_num = sp_data.get('disc_number', t.get('disc_number', 1)) + disc_num = sp_data.get('disc_number') or t.get('disc_number') or 1 if album_id: wishlist_album_disc_counts[album_id] = max( wishlist_album_disc_counts.get(album_id, 1), disc_num ) + if isinstance(album_val, dict): + existing_album_ctx = wishlist_album_context_map.get(album_id, {}) + if _album_context_richness(album_val) > _album_context_richness(existing_album_ctx): + wishlist_album_context_map[album_id] = dict(album_val) # Resolve album-level artist once per album (first track wins) if album_id not in wishlist_album_artist_map: _wl_source = t.get('source_info') or {} @@ -583,17 +961,20 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma _fb_name = track_info.get('artist', '') artist_ctx = {'name': _fb_name or 'Unknown Artist'} - # Construct minimal album context - # Ensure images are preserved (important for artwork) + # Construct a shared album context from the richest track in + # this album group so release_date/year and artwork do not + # vary per track and split folders. album_id = s_album.get('id', 'wishlist_album') + shared_album = wishlist_album_context_map.get(album_id_for_lookup, s_album) album_ctx = { 'id': album_id, - 'name': s_album.get('name'), - 'release_date': s_album.get('release_date', ''), - 'total_tracks': s_album.get('total_tracks', 1), - 'total_discs': wishlist_album_disc_counts.get(album_id, 1), - 'album_type': s_album.get('album_type', 'album'), - 'images': s_album.get('images', []) # Pass images array directly + 'name': shared_album.get('name') or s_album.get('name'), + 'release_date': shared_album.get('release_date', ''), + 'total_tracks': shared_album.get('total_tracks') or s_album.get('total_tracks', 1), + 'total_discs': wishlist_album_disc_counts.get(album_id_for_lookup, 1), + 'album_type': shared_album.get('album_type') or s_album.get('album_type', 'album'), + 'images': shared_album.get('images') or s_album.get('images', []), + 'artists': shared_album.get('artists') or s_album.get('artists', []), } track_info['_explicit_album_context'] = album_ctx diff --git a/core/downloads/monitor.py b/core/downloads/monitor.py index 3252f324..c4b645f5 100644 --- a/core/downloads/monitor.py +++ b/core/downloads/monitor.py @@ -34,6 +34,32 @@ _start_next_batch_of_downloads = None _orphaned_download_keys = None missing_download_executor = None download_orchestrator = None +_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet')) + + +def _download_id_key(download_id): + return f"download_id::{download_id}" if download_id else None + + +def _is_release_task(task): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or ti.get('username') + return username in _RELEASE_SOURCE_NAMES + + +def _lookup_live_info(task, live_transfers_lookup): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + download_id = task.get('download_id') + if _is_release_task(task): + by_id = live_transfers_lookup.get(_download_id_key(download_id)) + if by_id: + return by_id + + task_filename = task.get('filename') or ti.get('filename') + task_username = task.get('username') or ti.get('username') + if not task_filename or not task_username: + return None + return live_transfers_lookup.get(_make_context_key(task_username, task_filename)) def init( @@ -139,55 +165,64 @@ class WebUIDownloadMonitor: for task_id in download_batches[batch_id].get('queue', []): task = download_tasks.get(task_id) - if not task or task['status'] not in ['downloading', 'queued']: + if not task: + continue + release_recoverable = ( + _is_release_task(task) + and task.get('download_id') + and task.get('status') in ['failed', 'not_found'] + ) + if task['status'] not in ['downloading', 'queued'] and not release_recoverable: continue # Check for timeouts and errors - retries handled directly in _should_retry_task # If _should_retry_task returns True, it means retries were exhausted - retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) + retry_exhausted = False + if not release_recoverable: + retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) # Collect exhausted tasks to handle outside lock (prevents deadlock) if retry_exhausted: exhausted_tasks.append((batch_id, task_id)) - # ENHANCED: Check for successful completions (especially YouTube) - task_filename = task.get('filename') or task.get('track_info', {}).get('filename') - task_username = task.get('username') or task.get('track_info', {}).get('username') + # ENHANCED: Check for successful completions (especially YouTube). + # Release-style sources can report a completed audio file + # name that differs from the original indexer URL/title + # stored on the task, so prefer the stable download_id. + live_info = _lookup_live_info(task, live_transfers_lookup) - if task_filename and task_username: - lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) - - if live_info: - state = live_info.get('state', '') - # Trigger post-processing if download is completed successfully - # slskd uses compound states like 'Completed, Succeeded' - use substring matching - # Must exclude error states first (matching _build_batch_status_data's prioritized checking) - has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) - has_completion = ('Completed' in state or 'Succeeded' in state) - # Verify bytes actually transferred before trusting state string. - # slskd can report "Completed" before the full file is flushed to disk, - # or on connection drops that leave a partial file. - if has_completion and not has_error: - expected_size = live_info.get('size', 0) - transferred = live_info.get('bytesTransferred', 0) - if expected_size > 0 and transferred < expected_size: - if not task.get('_incomplete_warned'): - logger.debug(f"Monitor: {task_id} state={state} but bytes incomplete ({transferred}/{expected_size}) — waiting") - task['_incomplete_warned'] = True - continue - if has_completion and not has_error and task['status'] == 'downloading': - task.pop('_incomplete_warned', None) - # CRITICAL FIX: Transition to 'post_processing' HERE so downloads - # don't depend on browser polling to trigger post-processing. - # Previously, post-processing was only submitted by _build_batch_status_data - # (called from browser-polled endpoints), meaning closing the browser - # left tasks stuck in 'downloading' forever. - task['status'] = 'post_processing' - task['status_change_time'] = current_time - logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") - # Collect for handling outside the lock to prevent deadlock. - # _on_download_completed acquires tasks_lock which is non-reentrant. - completed_tasks.append((batch_id, task_id)) + if live_info: + state = live_info.get('state', '') + # Trigger post-processing if download is completed successfully + # slskd uses compound states like 'Completed, Succeeded' - use substring matching + # Must exclude error states first (matching _build_batch_status_data's prioritized checking) + has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) + has_completion = ('Completed' in state or 'Succeeded' in state) + # Verify bytes actually transferred before trusting state string. + # slskd can report "Completed" before the full file is flushed to disk, + # or on connection drops that leave a partial file. + if has_completion and not has_error: + expected_size = live_info.get('size', 0) + transferred = live_info.get('bytesTransferred', 0) + if expected_size > 0 and transferred < expected_size: + if not task.get('_incomplete_warned'): + logger.debug("Monitor: %s state=%s but bytes incomplete (%s/%s) - waiting", task_id, state, transferred, expected_size) + task['_incomplete_warned'] = True + continue + if has_completion and not has_error and ( + task['status'] == 'downloading' or release_recoverable + ): + task.pop('_incomplete_warned', None) + # CRITICAL FIX: Transition to 'post_processing' HERE so downloads + # don't depend on browser polling to trigger post-processing. + # Previously, post-processing was only submitted by _build_batch_status_data + # (called from browser-polled endpoints), meaning closing the browser + # left tasks stuck in 'downloading' forever. + task['status'] = 'post_processing' + task['status_change_time'] = current_time + logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") + # Collect for handling outside the lock to prevent deadlock. + # _on_download_completed acquires tasks_lock which is non-reentrant. + completed_tasks.append((batch_id, task_id)) # ---- All work below runs WITHOUT tasks_lock held ---- if globals().get('IS_SHUTTING_DOWN', False) or not self.monitoring: @@ -197,8 +232,21 @@ class WebUIDownloadMonitor: for op in deferred_ops: try: if op[0] == 'cancel_download': - _, download_id, username = op - logger.debug(f"[Deferred] Cancelling download: {download_id} from {username}") + # Issue #648 diagnostic — `op` now carries a trigger + # label (4-tuple, was 3-tuple) so the next log dump + # tells us WHICH path in `_should_retry_task` is + # firing for users seeing "Tidal downloads failed to + # start" mass-cancels. Label format pinned in commit + # message for grep-ability. + if len(op) >= 4: + _, download_id, username, trigger = op[0], op[1], op[2], op[3] + else: + _, download_id, username = op + trigger = 'unlabeled' + logger.info( + f"[CancelTrigger:monitor.{trigger}] download_id={download_id} " + f"username={username}" + ) run_async(download_orchestrator.cancel_download(download_id, username, remove=True)) logger.debug(f"[Deferred] Successfully cancelled download {download_id}") elif op[0] == 'cleanup_orphan': @@ -214,18 +262,28 @@ class WebUIDownloadMonitor: except Exception as e: logger.error(f"[Deferred] Error executing deferred operation {op[0]}: {e}") - # Handle completed downloads outside the lock to prevent deadlock - # (_on_download_completed acquires tasks_lock internally) + # Handle completed transfers outside the lock. The transfer engine's + # "complete" state only means the remote download finished; the + # post-processing worker still has to find, verify, tag, and move the + # file before it can report real batch success or failure. for batch_id, task_id in completed_tasks: try: # Submit post-processing worker (file move, tagging, AcoustID verification) # This makes batch downloads fully independent of browser polling. logger.info(f"[Monitor] Submitting post-processing worker for task {task_id}") missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id) - # Chain to next download in the batch queue - _on_download_completed(batch_id, task_id, success=True) except Exception as e: logger.error(f"[Monitor] Error handling completed task {task_id}: {e}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f'Post-processing could not be scheduled: {e}' + try: + _on_download_completed(batch_id, task_id, success=False) + except Exception as completion_error: + logger.error( + f"[Monitor] Error marking failed post-processing submit for task {task_id}: {completion_error}" + ) # Handle exhausted retry tasks outside the lock to prevent deadlock for batch_id, task_id in exhausted_tasks: try: @@ -285,7 +343,7 @@ class WebUIDownloadMonitor: for download in all_downloads: key = _make_context_key(download.username, download.filename) # Convert DownloadStatus to transfer dict format for monitor compatibility - live_transfers[key] = { + transfer_row = { 'id': download.id, 'filename': download.filename, 'username': download.username, @@ -295,6 +353,10 @@ class WebUIDownloadMonitor: 'bytesTransferred': download.transferred, 'averageSpeed': download.speed, } + live_transfers[key] = transfer_row + id_key = _download_id_key(download.id) + if id_key: + live_transfers[id_key] = transfer_row except Exception as yt_error: logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}") @@ -329,7 +391,7 @@ class WebUIDownloadMonitor: return False lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) + live_info = _lookup_live_info(task, live_transfers_lookup) if not live_info: # User-initiated manual pick — skip auto-retry. The status @@ -353,7 +415,8 @@ class WebUIDownloadMonitor: # Defer slskd cancel to outside the lock if task_username and download_id: - deferred_ops.append(('cancel_download', download_id, task_username)) + deferred_ops.append(('cancel_download', download_id, task_username, + 'not_in_live_transfers_90s')) # Mark current source as used (full filename to match worker format) if task_username and task_filename: @@ -424,7 +487,8 @@ class WebUIDownloadMonitor: # Defer slskd cancel to outside the lock if username and download_id: - deferred_ops.append(('cancel_download', download_id, username)) + deferred_ops.append(('cancel_download', download_id, username, + 'errored_state_retry')) # Mark current source as used to prevent retry loops # CRITICAL: Use full filename (not basename) to match worker's source_key format @@ -527,7 +591,8 @@ class WebUIDownloadMonitor: # Defer slskd cancel to outside the lock if username and download_id: - deferred_ops.append(('cancel_download', download_id, username)) + deferred_ops.append(('cancel_download', download_id, username, + 'queued_state_timeout')) # UNIFIED RETRY LOGIC: Handle timeout retry exactly like error retry # Mark current source as used to prevent retry loops @@ -615,7 +680,8 @@ class WebUIDownloadMonitor: # Defer slskd cancel to outside the lock if username and download_id: - deferred_ops.append(('cancel_download', download_id, username)) + deferred_ops.append(('cancel_download', download_id, username, + 'stuck_at_0pct_timeout')) # UNIFIED RETRY LOGIC: Handle 0% timeout retry exactly like error retry # Mark current source as used to prevent retry loops @@ -704,7 +770,8 @@ class WebUIDownloadMonitor: download_id = task.get('download_id') if username and download_id: - deferred_ops.append(('cancel_download', download_id, username)) + deferred_ops.append(('cancel_download', download_id, username, + 'unknown_state_no_progress_timeout')) if username and filename: used_sources = task.get('used_sources', set()) diff --git a/core/downloads/post_processing.py b/core/downloads/post_processing.py index 1b2d04d2..2c17e90f 100644 --- a/core/downloads/post_processing.py +++ b/core/downloads/post_processing.py @@ -19,9 +19,11 @@ from __future__ import annotations import logging import os +import shutil import time import traceback from dataclasses import dataclass +from difflib import SequenceMatcher from pathlib import Path from typing import Any, Callable, Optional @@ -34,7 +36,7 @@ from core.imports.context import ( get_import_original_search, normalize_import_context, ) -from core.imports.filename import extract_track_number_from_filename +from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata from core.metadata import enrichment as metadata_enrichment from core.runtime_state import ( download_tasks, @@ -46,6 +48,85 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +_AUDIO_EXTENSIONS = { + '.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif', + '.opus', '.ogg', '.m4a', '.aac', '.mp3', '.wma', +} + + +def _is_audio_file(path: str) -> bool: + return Path(str(path or '')).suffix.lower() in _AUDIO_EXTENSIONS + + +def _reject_non_audio_found_file(found_file: Optional[str], file_location: Optional[str]) -> tuple[Optional[str], Optional[str]]: + if found_file and not _is_audio_file(found_file): + logger.warning( + "[Post-Processing] Ignoring non-audio candidate found during file search: %s", + found_file, + ) + return None, None + return found_file, file_location + + +def _normalize_match_text(value: str) -> str: + return ''.join(ch.lower() for ch in str(value or '') if ch.isalnum()) + + +def _release_audio_match_score(path: str, expected_title: str, expected_artist: str) -> float: + parsed = parse_filename_metadata(path) + parsed_title = parsed.get('title') or Path(path).stem + parsed_artist = parsed.get('artist') or '' + expected_title_norm = _normalize_match_text(expected_title) + parsed_title_norm = _normalize_match_text(parsed_title) + if expected_title_norm and ( + expected_title_norm in parsed_title_norm or parsed_title_norm in expected_title_norm + ): + title_score = 1.0 + else: + title_score = SequenceMatcher( + None, + expected_title_norm, + parsed_title_norm, + ).ratio() + if expected_artist and parsed_artist: + artist_score = SequenceMatcher( + None, + _normalize_match_text(expected_artist), + _normalize_match_text(parsed_artist), + ).ratio() + return (title_score * 0.75) + (artist_score * 0.25) + return title_score + + +def _track_title_from_task(track_info: Any, context: Optional[dict]) -> str: + if isinstance(track_info, dict): + title = track_info.get('name') or track_info.get('title') + if title: + return str(title) + return get_import_clean_title(context or {}, default='') + + +def _copy_release_audio_to_transfer(source_path: str, transfer_dir: str) -> Optional[str]: + try: + src = Path(source_path) + if not src.exists() or not src.is_file(): + return None + dest_dir = Path(transfer_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / src.name + if dest.exists(): + stem, suffix = src.stem, src.suffix + counter = 1 + while dest.exists(): + dest = dest_dir / f"{stem}_release_{counter}{suffix}" + counter += 1 + shutil.copy2(src, dest) + return str(dest) + except Exception as exc: + logger.warning("[Post-Processing] Could not copy release audio to transfer: %s", exc) + return None + + @dataclass class PostProcessDeps: """Bundle of dependencies the post-processing worker needs. @@ -190,14 +271,71 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # CRITICAL FIX: For YouTube downloads, the filename in task is 'id||title' (metadata), # but the actual file on disk is 'Title.mp3'. We must ask the client for the real path. + # Torrent/usenet also use opaque "url||display" filenames. Their completed + # job may contain a full release, so choose the best matching audio file + # and copy it to transfer before importing instead of moving the client's + # original completed download. if (task.get('username') == 'youtube' or '||' in str(task_filename)) and not found_file: - logger.info(f"[Post-Processing] Detected YouTube download task: {task_id}") + logger.info(f"[Post-Processing] Detected engine-backed download task: {task_id}") try: # Query the download orchestrator for the status which contains the real file path # CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id actual_download_id = task.get('download_id') or task_id status = deps.run_async(deps.download_orchestrator.get_download_status(actual_download_id)) - if status and status.file_path: + if status and task.get('username') in ('torrent', 'usenet'): + audio_files = list(getattr(status, 'audio_files', None) or []) + if not audio_files and getattr(status, 'file_path', None): + audio_files = [status.file_path] + expected_title = _track_title_from_task(track_info, context) + expected_artist = '' + if isinstance(track_info, dict): + artists = track_info.get('artists', []) + if isinstance(artists, list) and artists: + first_artist = artists[0] + expected_artist = first_artist.get('name', '') if isinstance(first_artist, dict) else str(first_artist) + elif isinstance(artists, str): + expected_artist = artists + if not expected_artist and context: + artist_ctx = get_import_context_artist(context) + expected_artist = artist_ctx.get('name', '') if isinstance(artist_ctx, dict) else '' + + scored_files = [ + (_release_audio_match_score(path, expected_title, expected_artist), path) + for path in audio_files + if _is_audio_file(path) + ] + scored_files.sort(reverse=True) + if scored_files: + best_score, best_path = scored_files[0] + logger.info( + "[Post-Processing] Best %s release file for '%s': %s (score %.2f)", + task.get('username'), expected_title, best_path, best_score, + ) + if best_score >= 0.80: + copied_path = _copy_release_audio_to_transfer(best_path, transfer_dir) + if copied_path: + found_file = copied_path + file_location = 'download' + logger.info( + "[Post-Processing] Copied matched %s release file to transfer: %s", + task.get('username'), copied_path, + ) + else: + logger.warning( + "[Post-Processing] No %s release file met match threshold for '%s' (best %.2f)", + task.get('username'), expected_title, best_score, + ) + if not found_file: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"No matching audio file found in {task.get('username')} release" + ) + deps.on_download_completed(batch_id, task_id, False) + return + + if status and status.file_path and not found_file and task.get('username') not in ('torrent', 'usenet'): real_path = status.file_path if os.path.exists(real_path): # Determine if it's in download or transfer directory @@ -227,11 +365,12 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep logger.info(f"[Post-Processing] Resolved actual YouTube filename: {found_file} (Location: {file_location})") else: - logger.warning(f"[Post-Processing] YouTube status reported path but file missing: {real_path}") + logger.warning(f"[Post-Processing] Engine status reported path but file missing: {real_path}") else: - logger.warning(f"[Post-Processing] YouTube status returned no file_path for task {task_id}") + if not found_file: + logger.warning(f"[Post-Processing] Engine status returned no file_path for task {task_id}") except Exception as e: - logger.error(f"[Post-Processing] Failed to retrieve YouTube task status: {e}") + logger.error(f"[Post-Processing] Failed to retrieve engine task status: {e}") _file_search_max_retries = 5 for retry_count in range(_file_search_max_retries): @@ -257,6 +396,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep # Strategy 1: Try with original filename in both downloads and transfer logger.info("[Post-Processing] Strategy 1: Searching with original filename...") found_file, file_location = deps.find_completed_file(download_dir, task_filename, transfer_dir) + found_file, file_location = _reject_non_audio_found_file(found_file, file_location) if found_file: logger.info(f"[Post-Processing] Strategy 1 SUCCESS: Found file with original filename in {file_location}: {found_file}") @@ -269,7 +409,11 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep found_result = deps.find_completed_file(transfer_dir, expected_final_filename) if found_result and found_result[0]: found_file, file_location = found_result[0], 'transfer' - logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}") + found_file, file_location = _reject_non_audio_found_file(found_file, file_location) + if found_file: + logger.info(f"[Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}") + else: + logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename resolved to a non-audio file") else: logger.error("[Post-Processing] Strategy 2 FAILED: Expected final filename not found in transfer folder") elif not expected_final_filename: diff --git a/core/downloads/staging.py b/core/downloads/staging.py index 12de4a16..f7ca8906 100644 --- a/core/downloads/staging.py +++ b/core/downloads/staging.py @@ -34,9 +34,12 @@ from __future__ import annotations import logging import os +import re from dataclasses import dataclass from typing import Any, Callable +from core.imports.filename import extract_track_number_from_filename + # `shutil` and `SequenceMatcher` are imported inline inside try_staging_match() # to keep the lift byte-identical with the original web_server.py function body. @@ -50,6 +53,56 @@ from core.runtime_state import ( logger = logging.getLogger(__name__) +def _coerce_positive_int(value: Any, default: int = 0) -> int: + try: + coerced = int(str(value).split('/')[0]) + except (TypeError, ValueError): + return default + return coerced if coerced > 0 else default + + +def _staging_title_variants(title: Any, normalize: Callable[[str], str]) -> list[str]: + """Return conservative title variants for release-file matching. + + Torrent / usenet release files often encode featured artists in the + filename/title while streaming metadata keeps them in the artist credit. + Strip only feature/bonus noise here; keep version words like remix, + extended, live, acoustic, etc. so distinct recordings do not collapse. + """ + raw = str(title or '').strip() + if not raw: + return [] + + compacted_separators = re.sub(r'[_]+', ' ', raw) + compacted_separators = re.sub(r'\s+', ' ', compacted_separators).strip() + + without_feat = re.sub( + r'\s*[\(\[]\s*(?:feat\.?|ft\.?|featuring)\s+[^)\]]*[\)\]]', + '', + compacted_separators, + flags=re.IGNORECASE, + ) + without_feat = re.sub( + r'\s+(?:feat\.?|ft\.?|featuring)\s+.*$', + '', + without_feat, + flags=re.IGNORECASE, + ) + without_bonus = re.sub( + r'\s*[\(\[]\s*bonus\s+track\s*[\)\]]', + '', + without_feat, + flags=re.IGNORECASE, + ) + + variants: list[str] = [] + for candidate in (raw, compacted_separators, without_feat, without_bonus): + normalized = normalize(candidate) + if normalized and normalized not in variants: + variants.append(normalized) + return variants + + @dataclass class StagingDeps: """Bundle of cross-cutting deps the staging-match helper needs.""" @@ -58,6 +111,12 @@ class StagingDeps: get_staging_file_cache: Callable[[str], list] docker_resolve_path: Callable[[str], str] post_process_matched_download_with_verification: Callable + # Optional batch-field accessor. Returns ``download_batches[batch_id].get(field)`` + # when the runtime state is available, ``None`` otherwise. Injected so + # this module doesn't have to import from runtime_state directly — + # keeps the dep surface explicit and the function unit-testable + # without a live batch dict. + get_batch_field: Callable[[str, str], Any] = None # type: ignore[assignment] def try_staging_match(task_id, batch_id, track, deps: StagingDeps): @@ -80,19 +139,24 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): normalize = deps.matching_engine.normalize_string norm_title = normalize(track_title) norm_artist = normalize(track_artist) + title_variants = _staging_title_variants(track_title, normalize) or [norm_title] best_match = None best_score = 0.0 for sf in staging_files: - sf_norm_title = normalize(sf['title']) + sf_title_variants = _staging_title_variants(sf['title'], normalize) sf_norm_artist = normalize(sf['artist']) - if not sf_norm_title: + if not sf_title_variants: continue # Title similarity (primary) - title_sim = SequenceMatcher(None, norm_title, sf_norm_title).ratio() + title_sim = max( + SequenceMatcher(None, expected, candidate).ratio() + for expected in title_variants + for candidate in sf_title_variants + ) if title_sim < 0.80: continue @@ -141,20 +205,51 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): shutil.copy2(best_match['full_path'], dest_path) logger.info(f"[Staging] Copied to transfer: {dest_path}") - # Mark task as completed with staging context + # Mark task as completed with staging context. + # If the batch was populated by the torrent / usenet album-bundle + # flow, prefer that provenance label over generic 'staging' so the + # download history reflects the real source. The accessor is + # injected via StagingDeps so this module doesn't reach into + # runtime_state directly (see deps.get_batch_field docstring). + _provenance_override = None + if batch_id and deps.get_batch_field is not None: + try: + _provenance_override = deps.get_batch_field(batch_id, 'album_bundle_source') + except Exception as _exc: + logger.debug("get_batch_field failed: %s", _exc) + _provenance_username = _provenance_override or 'staging' + _private_album_bundle_staging = False + if batch_id and deps.get_batch_field is not None: + try: + _private_album_bundle_staging = bool( + deps.get_batch_field(batch_id, 'album_bundle_private_staging') + ) + except Exception as _exc: + logger.debug("get_batch_field failed: %s", _exc) with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'post_processing' download_tasks[task_id]['filename'] = dest_path - download_tasks[task_id]['username'] = 'staging' + download_tasks[task_id]['username'] = _provenance_username download_tasks[task_id]['staging_match'] = True + if _private_album_bundle_staging: + try: + os.remove(best_match['full_path']) + logger.debug("[Staging] Removed private album-bundle staging file: %s", best_match['full_path']) + except FileNotFoundError: + pass + except Exception as _exc: + logger.debug("[Staging] Could not remove private album-bundle staging file: %s", _exc) + # Run post-processing (tagging, AcoustID verification, path building) context_key = f"staging_{task_id}" with tasks_lock: track_info = download_tasks.get(task_id, {}).get('track_info', {}) if not isinstance(track_info, dict): track_info = {} + else: + track_info = dict(track_info) # Build spotify_artist / spotify_album context so post-processing can apply # the path template. Without these, _post_process_matched_download returns @@ -223,20 +318,37 @@ def try_staging_match(task_id, batch_id, track, deps: StagingDeps): ) has_clean_data = bool(track_title and track_artist and track_album_name) - track_number = ( - track_info.get('track_number', 0) or - getattr(track, 'track_number', 0) or 0 - ) - disc_number = ( - track_info.get('disc_number', 1) or - getattr(track, 'disc_number', 1) or 1 + file_track_number = ( + _coerce_positive_int(best_match.get('track_number'), 0) or + extract_track_number_from_filename(best_match.get('full_path', '')) ) + file_disc_number = _coerce_positive_int(best_match.get('disc_number'), 1) + if _private_album_bundle_staging: + track_number = file_track_number + disc_number = file_disc_number + else: + track_number = ( + _coerce_positive_int(track_info.get('track_number'), 0) or + _coerce_positive_int(track_info.get('trackNumber'), 0) or + _coerce_positive_int(getattr(track, 'track_number', 0), 0) or + file_track_number + ) + disc_number = ( + _coerce_positive_int(track_info.get('disc_number'), 0) or + _coerce_positive_int(track_info.get('discNumber'), 0) or + _coerce_positive_int(getattr(track, 'disc_number', 0), 0) or + file_disc_number + ) + track_info['track_number'] = track_number + track_info['disc_number'] = disc_number context = { 'track_info': track_info, 'spotify_artist': spotify_artist_ctx, 'spotify_album': spotify_album_ctx, 'original_search_result': { + 'username': _provenance_username, + 'filename': best_match.get('full_path', ''), 'title': track_title, 'artist': track_artist, 'spotify_clean_title': track_title, diff --git a/core/downloads/status.py b/core/downloads/status.py index c1599c5a..6e11a763 100644 --- a/core/downloads/status.py +++ b/core/downloads/status.py @@ -87,8 +87,10 @@ class StatusDeps: # Streaming sources the engine fallback applies to. Soulseek goes through # slskd's live_transfers path and must NOT hit the engine fallback. _STREAMING_SOURCE_NAMES = frozenset(( - 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', + 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', + 'torrent', 'usenet', )) +_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet')) # Keep these in sync with the engine plugins' state strings. _ENGINE_FAILURE_STATES = ('Errored', 'Failed', 'Rejected', 'TimedOut', 'Aborted') @@ -136,21 +138,12 @@ def _apply_engine_state_fallback( Mutates ``task`` in place (status / error_message) the same way the Soulseek branch does, so the next status poll sees the new state. - Submits post-processing on terminal success and fires - ``on_download_completed`` on terminal failure to free the worker - slot. + Submits post-processing on terminal success. Manual-pick failures + are completed here; automatic failures keep the existing retry path + in charge. """ if deps.download_orchestrator is None or deps.run_async is None: return - if task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing'): - return - # Scope this fallback to user-initiated manual picks. Auto attempts - # already flow through the live_transfers_lookup IF branch (the engine - # pre-populates non-Soulseek records via get_all_downloads), and on - # failure the monitor's existing retry path picks the next candidate. - # Marking auto attempts failed here would short-circuit that fallback. - if not task.get('_user_manual_pick'): - return download_id = task.get('download_id') if not download_id: return @@ -158,6 +151,17 @@ def _apply_engine_state_fallback( username = task.get('username') or ti.get('username') if username not in _STREAMING_SOURCE_NAMES: return + manual_pick = bool(task.get('_user_manual_pick')) + if not manual_pick and username not in _RELEASE_SOURCE_NAMES: + return + release_source = username in _RELEASE_SOURCE_NAMES + terminal_status = task.get('status') in ('completed', 'failed', 'cancelled', 'not_found', 'post_processing') + if terminal_status and not ( + release_source + and task.get('status') in ('failed', 'not_found') + and download_id + ): + return try: record = deps.run_async( @@ -189,6 +193,14 @@ def _apply_engine_state_fallback( return if any(s in state_str for s in _ENGINE_FAILURE_STATES): + if not manual_pick: + task_status['status'] = task.get('status', 'downloading') + task_status['progress'] = _engine_progress_pct(record) + logger.info( + "[Engine Fallback] Task %s engine reports '%s' for auto %s attempt; leaving retry handling to monitor", + task_id, state_str, username, + ) + return if task['status'] != 'failed': task['status'] = 'failed' err = getattr(record, 'error_message', None) or getattr(record, 'error', None) or '' @@ -247,6 +259,9 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d "playlist_id": batch.get('playlist_id'), # Include playlist_id for rehydration "playlist_name": batch.get('playlist_name'), # Include playlist_name for reference } + album_bundle = _build_album_bundle_status(batch) + if album_bundle: + response_data['album_bundle'] = album_bundle if response_data["phase"] == 'analysis': response_data['analysis_progress'] = { @@ -312,11 +327,23 @@ def build_batch_status_data(batch_id: str, batch: dict, live_transfers_lookup: d 'ui_state': task.get('ui_state', 'normal'), # normal|cancelling|cancelled 'playlist_id': task.get('playlist_id'), # For V2 system identification 'error_message': task.get('error_message'), # Surface failure reasons to UI + 'quarantine_entry_id': task.get('quarantine_entry_id'), 'has_candidates': bool(task.get('cached_candidates')), # Whether search found results (for clickable review) } _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} task_filename = task.get('filename') or _ti.get('filename') task_username = task.get('username') or _ti.get('username') + if ( + task_username in _RELEASE_SOURCE_NAMES + and task['status'] not in ['completed', 'cancelled', 'post_processing'] + and task.get('download_id') + ): + _apply_engine_state_fallback( + task_id, task, task_status, batch_id, deps, + ) + batch_tasks.append(task_status) + continue + if task_filename and task_username: lookup_key = deps.make_context_key(task_username, task_filename) @@ -609,7 +636,7 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: for bid, batch in download_batches.items(): queue = batch.get('queue', []) statuses = [download_tasks[tid]['status'] for tid in queue if tid in download_tasks] - batch_summaries.append({ + summary = { 'batch_id': bid, 'playlist_id': batch.get('playlist_id', ''), 'batch_name': batch.get('playlist_name') or batch.get('album_name') or '', @@ -620,7 +647,11 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: 'failed': sum(1 for s in statuses if s in ('failed', 'not_found', 'cancelled')), 'active': sum(1 for s in statuses if s in ('downloading', 'searching', 'post_processing')), 'queued': sum(1 for s in statuses if s in ('queued', 'pending')), - }) + } + album_bundle = _build_album_bundle_status(batch) + if album_bundle: + summary['album_bundle'] = album_bundle + batch_summaries.append(summary) return { 'success': True, @@ -629,3 +660,38 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict: 'batches': batch_summaries, 'timestamp': time.time(), } + + +def _build_album_bundle_status(batch: dict) -> dict: + """Return public batch-level status for torrent/Usenet album-bundle work.""" + state = batch.get('album_bundle_state') + if not state: + return {} + + status = { + 'state': state, + 'source': batch.get('album_bundle_source'), + 'release': batch.get('album_bundle_release'), + 'progress': batch.get('album_bundle_progress'), + 'progress_percent': _album_bundle_progress_percent( + batch.get('album_bundle_progress') + ), + 'speed': batch.get('album_bundle_speed'), + 'downloaded': batch.get('album_bundle_downloaded'), + 'size': batch.get('album_bundle_size'), + 'seeders': batch.get('album_bundle_seeders'), + 'grabs': batch.get('album_bundle_grabs'), + 'count': batch.get('album_bundle_count'), + } + return {key: value for key, value in status.items() if value is not None} + + +def _album_bundle_progress_percent(value: Any) -> int: + try: + progress = float(value) + except (TypeError, ValueError): + return 0 + + if progress <= 1: + progress *= 100 + return max(0, min(100, int(round(progress)))) diff --git a/core/downloads/task_worker.py b/core/downloads/task_worker.py index ebe5f716..77cb803f 100644 --- a/core/downloads/task_worker.py +++ b/core/downloads/task_worker.py @@ -25,12 +25,49 @@ import traceback from dataclasses import dataclass from typing import Any, Callable, Optional -from core.runtime_state import download_tasks, tasks_lock +from core.runtime_state import download_batches, download_tasks, tasks_lock from core.spotify_client import Track as SpotifyTrack logger = logging.getLogger(__name__) +def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]: + """Return a user-facing miss reason when per-track search should stop. + + Torrent / usenet / Soulseek album batches first download one private staged release, + then each track claims the matching staged file. If that claim fails after + the release is already staged, falling through to the normal per-track + search only retries release-level sources N times and can keep re-adding + the same torrent. Treat the staged release as authoritative for this pass. + """ + if not batch_id: + return None + + batch = download_batches.get(batch_id) + if not isinstance(batch, dict): + return None + + source = (batch.get('album_bundle_source') or '').lower() + mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower() + hybrid_first = '' + if mode == 'hybrid': + order = getattr(deps.download_orchestrator, 'hybrid_order', None) or [] + if order: + hybrid_first = str(order[0] or '').lower() + else: + hybrid_first = str(getattr(deps.download_orchestrator, 'hybrid_primary', '') or '').lower() + if ( + batch.get('album_bundle_private_staging') + and batch.get('album_bundle_state') == 'staged' + and not batch.get('album_bundle_partial') + and source in ('torrent', 'usenet', 'soulseek') + and (mode == source or (mode == 'hybrid' and hybrid_first == source)) + ): + return f'Track was not found in the staged {source} album release' + + return None + + @dataclass class TaskWorkerDeps: """Bundle of cross-cutting deps the per-task download worker needs.""" @@ -128,6 +165,21 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke # === STAGING CHECK: Check staging folder for existing file before searching === if deps.try_staging_match(task_id, batch_id, track): return + staging_miss_reason = _private_album_bundle_staging_miss_reason(batch_id, deps) + if staging_miss_reason: + logger.warning( + "[Modal Worker] %s for '%s'; skipping redundant per-track %s search", + staging_miss_reason, + track.name, + getattr(deps.download_orchestrator, 'mode', 'release-source'), + ) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'not_found' + download_tasks[task_id]['error_message'] = staging_miss_reason + if batch_id: + deps.on_download_completed(batch_id, task_id, False) + return # Initialize task state tracking (like GUI's parallel_search_tracking) with tasks_lock: @@ -147,6 +199,27 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke artist_name = track.artists[0] if track.artists else None track_name = track.name + release_queries = [] + try: + _download_mode = (getattr(deps.download_orchestrator, 'mode', '') or '').lower() + _track_album = (getattr(track, 'album', '') or '').strip() + _track_title = (getattr(track, 'name', '') or '').strip() + _track_artists = list(getattr(track, 'artists', []) or []) + _first_artist = _track_artists[0] if _track_artists else '' + _primary_artist = ( + (_first_artist.get('name', '') if isinstance(_first_artist, dict) else str(_first_artist)) + or '' + ).strip() + if ( + _download_mode in ('torrent', 'usenet') + and _primary_artist + and _track_album + and _track_album.lower() not in ('unknown album', _track_title.lower()) + ): + release_queries.append(f"{_primary_artist} {_track_album}".strip()) + except Exception as _release_query_exc: + logger.debug("[Modal Worker] release query hint failed: %s", _release_query_exc) + # Start with matching engine queries search_queries = deps.matching_engine.generate_download_queries(track) @@ -175,8 +248,14 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke if cleaned_name and cleaned_name.lower() != track_name.lower(): legacy_queries.append(cleaned_name.strip()) - # Combine enhanced queries with legacy fallbacks - all_queries = search_queries + legacy_queries + # Combine enhanced queries with legacy fallbacks. + # + # Torrent / usenet can use full album releases as a fallback for + # single-track requests, but trying the album release first makes + # playlist batches download whole albums before checking whether a + # track-shaped release exists. Keep release queries last so singles + # stay light when the indexer has a direct result. + all_queries = search_queries + legacy_queries + release_queries # Remove duplicates while preserving order unique_queries = [] @@ -209,8 +288,30 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke logger.debug(f"About to call soulseek search for task {task_id}") try: + # Hybrid + album-context batches must skip torrent / usenet during + # the per-track loop — they're release-level sources, can't match + # individual tracks meaningfully, and album-bundle handling only + # fires in single-source mode (see core/downloads/master.py). The + # exclusion lets the hybrid chain fall through to per-track- + # compatible sources (soulseek / streaming) instead of attempting + # N redundant Prowlarr searches that all download the same album + # torrent and rely on the auto-import sweep to clean up. + _exclude_for_hybrid_album = None + try: + _batch_is_album = False + if batch_id: + from core.runtime_state import download_batches as _db + _b = _db.get(batch_id) + if isinstance(_b, dict): + _batch_is_album = bool(_b.get('is_album_download')) + if _batch_is_album and getattr(deps.download_orchestrator, 'mode', '') == 'hybrid': + _exclude_for_hybrid_album = ['torrent', 'usenet'] + except Exception as _exc_filter_err: + logger.debug("[Modal Worker] album-source-exclusion check failed: %s", _exc_filter_err) # Perform search with timeout - tracks_result, _ = deps.run_async(deps.download_orchestrator.search(query, timeout=30)) + tracks_result, _ = deps.run_async(deps.download_orchestrator.search( + query, timeout=30, exclude_sources=_exclude_for_hybrid_album, + )) logger.debug(f"Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results") # CRITICAL: Check cancellation immediately after search returns @@ -294,7 +395,7 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke source_clients = { name: orch.client(name) for name in ('soulseek', 'youtube', 'tidal', 'qobuz', - 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') + 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon') } # The orchestrator tried sources in order but stopped at the first with results. diff --git a/core/downloads/validation.py b/core/downloads/validation.py index 17171535..339d0e8e 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -9,6 +9,7 @@ import logging import re from config.settings import config_manager +from core.imports.file_integrity import resolve_duration_tolerance logger = logging.getLogger(__name__) @@ -24,6 +25,20 @@ def init(matching_engine_obj, download_orchestrator_obj): download_orchestrator = download_orchestrator_obj +def _torrent_usenet_artist_is_fallback(result): + """True when a release result has no parsed artist, only indexer filler.""" + if getattr(result, 'username', None) not in ('torrent', 'usenet'): + return False + artist = (getattr(result, 'artist', None) or '').strip() + if not artist: + return True + metadata = getattr(result, '_source_metadata', None) or {} + indexer = str(metadata.get('indexer') or '').strip() + if artist.lower() in ('torrent', 'usenet'): + return True + return bool(indexer and artist.lower() == indexer.lower()) + + def filter_soundcloud_previews(results, expected_track): """Drop SoundCloud preview snippets so they never reach the cache, the modal, or the auto-download attempt. @@ -61,6 +76,24 @@ def filter_soundcloud_previews(results, expected_track): return [r for r in results if not _is_preview(r)] +def _duration_tolerance_seconds(expected_duration_ms): + override = resolve_duration_tolerance( + config_manager.get('post_processing.duration_tolerance_seconds', 0) + ) + if override is not None: + return override + expected_seconds = expected_duration_ms / 1000.0 + return 5.0 if expected_seconds > 600.0 else 3.0 + + +def _duration_mismatch_exceeds_integrity_tolerance(expected_duration_ms, candidate_duration_ms): + if not expected_duration_ms or not candidate_duration_ms: + return False + tolerance = _duration_tolerance_seconds(expected_duration_ms) + drift = abs((candidate_duration_ms / 1000.0) - (expected_duration_ms / 1000.0)) + return drift > tolerance + + def get_valid_candidates(results, spotify_track, query): """ This function is a direct port from sync.py. It scores and filters @@ -78,8 +111,12 @@ def get_valid_candidates(results, spotify_track, query): return [] # Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer, SoundCloud) return structured API results - # with proper artist/title metadata — score using the same matching engine as Soulseek - _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud") + # with proper artist/title metadata — score using the same matching engine as Soulseek. + # Torrent / usenet results also belong here: their filename field is a download URL, not + # a slskd-style ``Artist/Album/Track.flac`` path, so the Soulseek matcher would extract + # garbage segments from it. Routing them through the streaming path means score_track_match + # reads ``r.title`` and ``r.artist`` directly (which the torrent/usenet projections pre-fill). + _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud", "amazon", "torrent", "usenet") if results[0].username in _streaming_sources: source_label = results[0].username.replace('_dl', '').title() expected_artists = spotify_track.artists if spotify_track else [] @@ -98,17 +135,70 @@ def get_valid_candidates(results, spotify_track, query): expected_is_version = any(kw in expected_title_lower for kw in _version_keywords) scored = [] + _strict_duration_sources = {'tidal', 'qobuz', 'hifi', 'deezer_dl', 'amazon'} for r in results: - # Score using matching engine's generic scorer (same weights as Soulseek) + if ( + r.username in _strict_duration_sources + and _duration_mismatch_exceeds_integrity_tolerance(expected_duration, r.duration or 0) + ): + logger.info( + "[%s] Rejecting candidate due to duration mismatch before download: " + "expected %.1fs, candidate %.1fs", + source_label, + expected_duration / 1000.0, + (r.duration or 0) / 1000.0, + ) + continue + + # Score using matching engine's generic scorer (same weights as Soulseek). + # Torrent/usenet release projections sometimes only have the indexer name + # in the artist field when a title did not parse as "Artist - Release". + # Treat that as unknown artist, not as a real mismatch. + has_only_fallback_artist = _torrent_usenet_artist_is_fallback(r) + candidate_artists = [] if has_only_fallback_artist else ([r.artist] if r.artist else []) confidence, match_type = matching_engine.score_track_match( source_title=expected_title, source_artists=expected_artists, source_duration_ms=expected_duration, candidate_title=r.title or '', - candidate_artists=[r.artist] if r.artist else [], + candidate_artists=candidate_artists, candidate_duration_ms=r.duration or 0, ) + # Album-name fallback for torrent / usenet per-track results. + # + # When this fallback runs: hybrid mode + non-album batch (single + # track wishlist / playlist of singles). Album-context batches + # never reach here — the album-bundle gate in + # core/downloads/album_bundle_dispatch.py engages the bulk- + # download flow in single-source mode, and the hybrid chain + # filter in core/downloads/task_worker.py strips torrent / + # usenet from album batches in hybrid mode. What's left is the + # single-track-in-hybrid case where a user is searching for one + # track and the only torrent / usenet result is the album that + # contains it. + # + # Without this fallback, "Luther (with SZA)" against a + # candidate titled "GNX (2024) [FLAC]" scores ~0 on track-title + # alone — even though the album torrent does in fact contain + # the wanted track. Scoring the candidate title against the + # wanted track's ALBUM name and taking the max gives album- + # level releases a fair shot. The Auto-Import sweep then picks + # the right file out of the downloaded album folder. + expected_album = getattr(spotify_track, 'album', None) if spotify_track else None + if r.username in ('torrent', 'usenet') and expected_album: + album_conf, _ = matching_engine.score_track_match( + source_title=expected_album, + source_artists=expected_artists, + source_duration_ms=0, # albums don't have one duration + candidate_title=r.title or '', + candidate_artists=candidate_artists, + candidate_duration_ms=0, + ) + if album_conf > confidence: + confidence = album_conf + match_type = 'album_release' + # Version detection penalty — reject live/remix/acoustic when expecting original r_title_lower = (r.title or '').lower() is_wrong_version = False @@ -129,8 +219,10 @@ def get_valid_candidates(results, spotify_track, query): # Artist gate — streaming APIs (Tidal/Qobuz/HiFi/Deezer) have reliable metadata, # so "My Will" by "B. Starr" should never match expected "B小町". - # Skip for YouTube — artist is parsed from video titles and often unreliable. - if r.username != 'youtube': + # YouTube stays excluded because video-title parsing is unreliable. + # Torrent/usenet must also pass this gate so title-only matches + # from the wrong artist do not get downloaded. + if r.username != 'youtube' and not has_only_fallback_artist: from difflib import SequenceMatcher import re as _re _cand_artist_raw = r.artist or '' @@ -162,6 +254,16 @@ def get_valid_candidates(results, spotify_track, query): # so falling to SequenceMatcher means the strings are genuinely # different. 0.5 gives a safer buffer without blocking real # matches that would have scored above 0.85 anyway. + if r.username in ('torrent', 'usenet') and _best_artist < 0.5: + logger.info( + "[%s] Rejecting candidate due to artist mismatch: " + "expected=%s candidate=%r title=%r", + source_label, + list(expected_artists), + _cand_artist_raw, + r.title or '', + ) + continue if _best_artist < 0.5 and confidence < 0.85: continue diff --git a/core/hifi_client.py b/core/hifi_client.py index 8fa51cdd..242983ea 100644 --- a/core/hifi_client.py +++ b/core/hifi_client.py @@ -23,6 +23,8 @@ import re import uuid import time import shutil +import json +import base64 import subprocess import threading from typing import List, Optional, Dict, Any, Tuple @@ -75,6 +77,13 @@ HLS_QUALITY_MAP = { HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"') +TRACK_ENDPOINT_QUALITY_MAP = { + 'hires': 'HI_RES_LOSSLESS', + 'lossless': 'LOSSLESS', + 'high': 'HIGH', + 'low': 'LOW', +} + # Default public hifi-api instances (ordered by preference) DEFAULT_INSTANCES = [ 'https://triton.squid.wtf', @@ -244,6 +253,122 @@ class HiFiClient(DownloadSourcePlugin): return data.get('version') or data.get('data', {}).get('version') return None + @staticmethod + def _extract_manifest_uri(data: Any) -> Optional[str]: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + attrs = inner.get('data', {}).get('attributes', {}) + return attrs.get('uri') + except (AttributeError, KeyError): + return None + + @staticmethod + def _extract_track_manifest_urls(data: Any) -> List[str]: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + manifest_b64 = inner.get('manifest') + if not manifest_b64: + return [] + manifest = json.loads(base64.b64decode(manifest_b64)) + if manifest.get('encryptionType') not in (None, 'NONE'): + return [] + urls = manifest.get('urls') or [] + return [url for url in urls if isinstance(url, str) and url] + except Exception as e: + logger.debug("Failed to extract legacy HiFi track manifest URLs: %s", e) + return [] + + @staticmethod + def _extension_from_track_manifest(data: Any, fallback: str) -> str: + try: + inner = data.get('data', data) if isinstance(data, dict) else data + manifest = json.loads(base64.b64decode(inner.get('manifest') or '')) + mime = (manifest.get('mimeType') or '').lower() + codecs = (manifest.get('codecs') or '').lower() + if 'flac' in mime or 'flac' in codecs: + return 'flac' + if 'mp4' in mime or 'aac' in codecs: + return 'm4a' + except Exception as e: + logger.debug("Failed to infer legacy HiFi track manifest extension: %s", e) + return fallback + + def check_instance_capabilities(self, url: str, timeout: int = 5) -> Dict[str, Any]: + """Probe one public HiFi instance using the endpoints SoulSync needs.""" + entry = { + 'url': url, + 'status': 'unknown', + 'version': None, + 'can_search': False, + 'can_download': False, + } + try: + root = self.session.get( + f'{url}/', + timeout=timeout, + headers={'Accept': 'application/json'}, + ) + if not root.ok: + entry['status'] = f'error (HTTP {root.status_code})' + return entry + + data = root.json() + entry['version'] = data.get('version') or data.get('data', {}).get('version') + entry['status'] = 'online' + + search = self.session.get( + f'{url}/search/', + params={'s': 'test', 'limit': 1}, + timeout=timeout, + ) + entry['can_search'] = search.ok + + manifest = self.session.get( + f'{url}/trackManifests/', + params={ + 'id': '1550546', + 'formats': 'FLAC', + 'usage': 'DOWNLOAD', + 'manifestType': 'HLS', + 'adaptive': 'true', + 'uriScheme': 'HTTPS', + }, + timeout=timeout, + ) + entry['can_download'] = ( + manifest.ok + and bool(self._extract_manifest_uri(manifest.json())) + ) + if not manifest.ok: + entry['download_error'] = f'HTTP {manifest.status_code}' + elif not entry['can_download']: + legacy = self.session.get( + f'{url}/track/', + params={'id': '1550546', 'quality': 'LOSSLESS'}, + timeout=timeout, + ) + entry['can_download'] = ( + legacy.ok + and bool(self._extract_track_manifest_urls(legacy.json())) + ) + if not legacy.ok: + entry['download_error'] = f'HTTP {legacy.status_code}' + elif not entry['can_download']: + entry['download_error'] = 'No playable manifest URL' + else: + entry['download_probe'] = 'track' + else: + entry['download_probe'] = 'trackManifests' + except http_requests.exceptions.SSLError: + entry['status'] = 'ssl_error' + except http_requests.exceptions.ConnectTimeout: + entry['status'] = 'timeout' + except http_requests.exceptions.ConnectionError: + entry['status'] = 'offline' + except Exception as e: + entry['status'] = f'error ({type(e).__name__})' + return entry + def search_tracks(self, title: str = None, artist: str = None, album: str = None, limit: int = 20) -> List[Dict]: params = {'limit': limit} @@ -433,15 +558,12 @@ class HiFiClient(DownloadSourcePlugin): data = self._api_get('/trackManifests/', params=params, timeout=20) if not data: - return None + return self._get_legacy_track_manifest(track_id, quality) - try: - inner = data.get('data', data) if isinstance(data, dict) else data - attrs = inner.get('data', {}).get('attributes', {}) - uri = attrs.get('uri') - except (AttributeError, KeyError) as e: - logger.warning(f"Failed to extract playlist URI from manifest response: {e}") - return None + uri = self._extract_manifest_uri(data) + if uri is None: + logger.warning("Failed to extract playlist URI from manifest response") + return self._get_legacy_track_manifest(track_id, quality) if not uri: logger.warning(f"No playlist URI in manifest for track {track_id}") @@ -488,6 +610,28 @@ class HiFiClient(DownloadSourcePlugin): 'quality': quality, } + def _get_legacy_track_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]: + q_info = HLS_QUALITY_MAP.get(quality, HLS_QUALITY_MAP['lossless']) + api_quality = TRACK_ENDPOINT_QUALITY_MAP.get(quality, 'LOSSLESS') + data = self._api_get('/track/', params={'id': track_id, 'quality': api_quality}, timeout=20) + if not data: + return None + + direct_urls = self._extract_track_manifest_urls(data) + if not direct_urls: + logger.warning(f"No playable URL in legacy HiFi manifest for track {track_id}") + return None + + extension = self._extension_from_track_manifest(data, q_info['extension']) + logger.info(f"HiFi legacy track manifest for track {track_id}: " + f"{len(direct_urls)} direct URL(s) ({quality})") + return { + 'direct_urls': direct_urls, + 'extension': extension, + 'codec': q_info['codec'], + 'quality': quality, + } + def _demux_flac(self, input_path: Path, output_path: Path) -> None: ffmpeg = shutil.which('ffmpeg') if not ffmpeg: @@ -619,7 +763,13 @@ class HiFiClient(DownloadSourcePlugin): return None manifest_info = self._get_hls_manifest(track_id, quality=q_key) - if not manifest_info or not manifest_info.get('segment_uris'): + if ( + not manifest_info + or ( + not manifest_info.get('segment_uris') + and not manifest_info.get('direct_urls') + ) + ): logger.warning(f"No HLS manifest at quality {q_key}, trying next") continue @@ -628,16 +778,17 @@ class HiFiClient(DownloadSourcePlugin): out_filename = f"{safe_name}.{extension}" out_path = self.download_path / out_filename - is_flac = q_key in ('hires', 'lossless') + is_direct = bool(manifest_info.get('direct_urls')) + is_flac = q_key in ('hires', 'lossless') and not is_direct intermediate_path = out_path.with_suffix('.m4a') if is_flac else out_path try: init_uri = manifest_info.get('init_uri') - segment_uris = manifest_info['segment_uris'] + segment_uris = manifest_info.get('segment_uris') or manifest_info.get('direct_urls') or [] total_segments = len(segment_uris) + (1 if init_uri else 0) logger.info(f"Downloading from HiFi ({q_key}): {out_filename} " - f"({total_segments} segments)") + f"({total_segments} {'URL(s)' if is_direct else 'segments'})") downloaded = 0 speed_start = time.time() diff --git a/core/image_cache.py b/core/image_cache.py new file mode 100644 index 00000000..040c1126 --- /dev/null +++ b/core/image_cache.py @@ -0,0 +1,342 @@ +"""Disk-backed image cache for browser-facing artwork URLs.""" + +from __future__ import annotations + +import hashlib +import mimetypes +import os +import sqlite3 +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Optional +from urllib.parse import urlparse + +import requests + +from config.settings import config_manager +from core.metadata.artwork import is_internal_image_host +from utils.logging_config import get_logger + +logger = get_logger("image_cache") + +DEFAULT_TTL_SECONDS = 30 * 24 * 60 * 60 +DEFAULT_FAILED_TTL_SECONDS = 6 * 60 * 60 +DEFAULT_MAX_DOWNLOAD_BYTES = 15 * 1024 * 1024 + + +class ImageCacheError(Exception): + """Raised when an image cannot be served from the cache.""" + + +@dataclass +class CachedImage: + key: str + path: Path + mime_type: str + size: int + status: str + + +class ImageCache: + def __init__( + self, + cache_dir: str | os.PathLike[str], + *, + ttl_seconds: int = DEFAULT_TTL_SECONDS, + failed_ttl_seconds: int = DEFAULT_FAILED_TTL_SECONDS, + max_download_bytes: int = DEFAULT_MAX_DOWNLOAD_BYTES, + fetcher: Optional[Callable[..., requests.Response]] = None, + ): + self.cache_dir = Path(cache_dir) + self.ttl_seconds = int(ttl_seconds) + self.failed_ttl_seconds = int(failed_ttl_seconds) + self.max_download_bytes = int(max_download_bytes) + self.fetcher = fetcher or requests.get + self.db_path = self.cache_dir / "image_cache.sqlite3" + self._db_lock = threading.RLock() + self._key_locks: dict[str, threading.Lock] = {} + self._key_locks_lock = threading.Lock() + self.cache_dir.mkdir(parents=True, exist_ok=True) + self._init_db() + + def cache_url_for(self, url: str | None) -> str | None: + """Register a URL and return its browser-facing cached path.""" + if not url: + return None + if str(url).startswith("/api/image-cache/"): + return str(url) + if not self.is_cacheable_url(str(url)): + return str(url) + + key = self.key_for_url(str(url)) + now = time.time() + with self._db_lock: + with self._connect() as conn: + conn.execute( + """ + INSERT INTO image_cache + (key, original_url, status, created_at, updated_at, last_accessed, + expires_at, size, mime_type, file_path, last_error) + VALUES (?, ?, 'pending', ?, ?, ?, 0, 0, '', '', '') + ON CONFLICT(key) DO UPDATE SET + original_url=excluded.original_url, + last_accessed=excluded.last_accessed + """, + (key, str(url), now, now, now), + ) + return f"/api/image-cache/{key}" + + def get(self, key: str) -> CachedImage: + row = self._get_row(key) + if not row: + raise ImageCacheError("Image cache key not found") + return self.get_url(row["original_url"]) + + def get_url(self, url: str) -> CachedImage: + if not self.is_cacheable_url(url): + raise ImageCacheError("URL is not cacheable") + + key = self.key_for_url(url) + lock = self._lock_for_key(key) + with lock: + row = self._get_row(key) + now = time.time() + if row and row["status"] == "ok" and row["file_path"]: + path = Path(row["file_path"]) + if path.exists(): + self._touch(key, now) + if float(row["expires_at"] or 0) > now: + return CachedImage(key, path, row["mime_type"] or "image/jpeg", int(row["size"] or 0), "hit") + + try: + return self._fetch_and_store(url, key, now) + except Exception as exc: + if row and row["status"] == "ok" and row["file_path"]: + stale_path = Path(row["file_path"]) + if stale_path.exists(): + logger.warning("Serving stale cached image for %s after refresh failed: %s", key, exc) + self._record_error(key, str(exc), now, keep_status=True) + return CachedImage( + key, + stale_path, + row["mime_type"] or "image/jpeg", + int(row["size"] or 0), + "stale", + ) + self._record_error(key, str(exc), now) + raise ImageCacheError(str(exc)) from exc + + @staticmethod + def key_for_url(url: str) -> str: + return hashlib.sha256(url.encode("utf-8")).hexdigest() + + @staticmethod + def is_cacheable_url(url: str) -> bool: + try: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"}: + return False + if parsed.username or parsed.password: + return False + if not parsed.hostname: + return False + return True + except Exception: + return False + + def _fetch_and_store(self, url: str, key: str, now: float) -> CachedImage: + if not self._is_fetch_allowed(url): + raise ImageCacheError("Image host is not allowed") + + response = self.fetcher( + url, + timeout=10, + stream=True, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ), + "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "Referer": "https://www.deezer.com/", + }, + ) + try: + if response.status_code != 200: + raise ImageCacheError(f"Upstream image returned HTTP {response.status_code}") + + mime_type = (response.headers.get("Content-Type") or "image/jpeg").split(";", 1)[0].strip() + if not mime_type.startswith("image/"): + raise ImageCacheError(f"Upstream response is not an image: {mime_type}") + + declared_size = response.headers.get("Content-Length") + try: + if declared_size and int(declared_size) > self.max_download_bytes: + raise ImageCacheError("Image exceeds configured size limit") + except ValueError: + pass + + ext = mimetypes.guess_extension(mime_type) or ".img" + if ext == ".jpe": + ext = ".jpg" + path = self._path_for_key(key, ext) + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + + total = 0 + try: + with open(tmp_path, "wb") as handle: + for chunk in response.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + total += len(chunk) + if total > self.max_download_bytes: + raise ImageCacheError("Image exceeds configured size limit") + handle.write(chunk) + except Exception: + try: + tmp_path.unlink(missing_ok=True) + except Exception as cleanup_exc: + logger.debug("image_cache tmp cleanup failed: %s", cleanup_exc) + raise + + if total <= 0: + raise ImageCacheError("Image response was empty") + + os.replace(tmp_path, path) + expires_at = now + self.ttl_seconds + with self._db_lock: + with self._connect() as conn: + conn.execute( + """ + INSERT INTO image_cache + (key, original_url, status, created_at, updated_at, last_accessed, + expires_at, size, mime_type, file_path, last_error) + VALUES (?, ?, 'ok', ?, ?, ?, ?, ?, ?, ?, '') + ON CONFLICT(key) DO UPDATE SET + original_url=excluded.original_url, + status='ok', + updated_at=excluded.updated_at, + last_accessed=excluded.last_accessed, + expires_at=excluded.expires_at, + size=excluded.size, + mime_type=excluded.mime_type, + file_path=excluded.file_path, + last_error='' + """, + (key, url, now, now, now, expires_at, total, mime_type, str(path)), + ) + return CachedImage(key, path, mime_type, total, "miss") + finally: + response.close() + + def _path_for_key(self, key: str, extension: str) -> Path: + return self.cache_dir / key[:2] / key[2:4] / f"{key}{extension}" + + def _is_fetch_allowed(self, url: str) -> bool: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"}: + return False + if parsed.username or parsed.password: + return False + if not parsed.hostname: + return False + + # Internal hosts are explicitly supported because Plex/Jellyfin/Navidrome + # artwork often lives behind Docker/LAN-only URLs. Public hosts are allowed + # as image-only responses with size limits. + return bool(parsed.hostname) or is_internal_image_host(url) + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init_db(self) -> None: + with self._db_lock: + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS image_cache ( + key TEXT PRIMARY KEY, + original_url TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + last_accessed REAL NOT NULL, + expires_at REAL NOT NULL DEFAULT 0, + size INTEGER NOT NULL DEFAULT 0, + mime_type TEXT NOT NULL DEFAULT '', + file_path TEXT NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '' + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_image_cache_accessed ON image_cache(last_accessed)") + + def _get_row(self, key: str) -> Optional[sqlite3.Row]: + with self._db_lock: + with self._connect() as conn: + return conn.execute("SELECT * FROM image_cache WHERE key = ?", (key,)).fetchone() + + def _touch(self, key: str, now: float) -> None: + with self._db_lock: + with self._connect() as conn: + conn.execute("UPDATE image_cache SET last_accessed = ? WHERE key = ?", (now, key)) + + def _record_error(self, key: str, error: str, now: float, *, keep_status: bool = False) -> None: + status_sql = "status" if keep_status else "'failed'" + with self._db_lock: + with self._connect() as conn: + conn.execute( + f""" + UPDATE image_cache + SET status = {status_sql}, + updated_at = ?, + last_accessed = ?, + expires_at = ?, + last_error = ? + WHERE key = ? + """, + (now, now, now + self.failed_ttl_seconds, error[:500], key), + ) + + def _lock_for_key(self, key: str) -> threading.Lock: + with self._key_locks_lock: + lock = self._key_locks.get(key) + if lock is None: + lock = threading.Lock() + self._key_locks[key] = lock + return lock + + +_image_cache: Optional[ImageCache] = None +_image_cache_lock = threading.Lock() + + +def get_image_cache() -> ImageCache: + global _image_cache + with _image_cache_lock: + if _image_cache is None: + cache_dir = config_manager.get("image_cache.path", "storage/image_cache") + if not os.path.isabs(cache_dir): + cache_dir = str(config_manager.base_dir / cache_dir) + _image_cache = ImageCache( + cache_dir, + ttl_seconds=int(config_manager.get("image_cache.ttl_seconds", DEFAULT_TTL_SECONDS)), + failed_ttl_seconds=int(config_manager.get("image_cache.failed_ttl_seconds", DEFAULT_FAILED_TTL_SECONDS)), + max_download_bytes=int(config_manager.get("image_cache.max_download_mb", 15)) * 1024 * 1024, + ) + return _image_cache + + +def cached_image_url(url: str | None) -> str | None: + if not url or config_manager.get("image_cache.enabled", True) is False: + return url + try: + return get_image_cache().cache_url_for(url) + except Exception as exc: + logger.debug("image cache registration failed: %s", exc) + return url diff --git a/core/imports/file_integrity.py b/core/imports/file_integrity.py index 82ced0d1..a0ebb18a 100644 --- a/core/imports/file_integrity.py +++ b/core/imports/file_integrity.py @@ -52,6 +52,37 @@ _DEFAULT_LENGTH_TOLERANCE_S = 3.0 _LENGTH_TOLERANCE_LONG_TRACK_S = 5.0 _LONG_TRACK_THRESHOLD_S = 600.0 # 10 minutes +# Upper bound for the user-configurable override. Anything past 60s +# means the check is effectively off — cap defends against accidental +# nonsense like 9999 making logs misleading. Users who genuinely want +# to disable the check can set 60. +_MAX_USER_TOLERANCE_S = 60.0 + + +def resolve_duration_tolerance(value: Any) -> Optional[float]: + """Coerce a user-configured tolerance value to a float override. + + Returns: + - None when value is missing / 0 / negative / unparseable, so + callers fall back to the auto-scaled defaults (3s/5s). + - float in (0, _MAX_USER_TOLERANCE_S] when value is a positive + numeric string or float — clamped to the upper bound. + + Pure helper. No I/O. Drives the `length_tolerance_s` override on + `check_audio_integrity`. + """ + if value is None: + return None + try: + parsed = float(value) + except (TypeError, ValueError): + return None + if parsed <= 0: + return None + if parsed > _MAX_USER_TOLERANCE_S: + return _MAX_USER_TOLERANCE_S + return parsed + @dataclass class IntegrityResult: diff --git a/core/imports/guards.py b/core/imports/guards.py index 657576f0..f4519a5d 100644 --- a/core/imports/guards.py +++ b/core/imports/guards.py @@ -29,8 +29,22 @@ def _get_config_manager(): return config_manager -def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None) -> str: - """Move a file to the quarantine folder and write a metadata sidecar.""" +def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None, *, trigger: str = "unknown") -> str: + """Move a file to the quarantine folder and write a metadata sidecar. + + `trigger` identifies which check fired (`integrity` / `acoustid` / + `bit_depth` / `unknown`) and is persisted in the sidecar so + one-click Approve can set the matching `_skip_quarantine_check` + bypass when re-running the pipeline. + + Sidecar also persists a JSON-safe snapshot of the full `context` + dict via `serialize_quarantine_context`, enabling in-place approve + without losing the matched-track metadata. Legacy sidecars (written + before this expansion) lack the `context` field — Approve falls + back to `recover_to_staging` for those. + """ + from core.imports.quarantine import serialize_quarantine_context + download_dir = _get_config_manager().get("soulseek.download_path", "./downloads") quarantine_dir = Path(download_dir) / "ss_quarantine" quarantine_dir.mkdir(parents=True, exist_ok=True) @@ -56,6 +70,8 @@ def move_to_quarantine(file_path: str, context: dict, reason: str, automation_en "expected_track": get_import_clean_title(context, default=original_search.get("title", "Unknown")), "expected_artist": get_import_clean_artist(context, default=(artist_context.get("name", "") if isinstance(artist_context, dict) else "Unknown")), "context_key": context.get("context_key", "unknown"), + "trigger": trigger, + "context": serialize_quarantine_context(context), } try: diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index d7b2b7d0..2f2f177d 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -32,9 +32,10 @@ from core.imports.context import ( get_import_track_info, normalize_import_context, ) -from core.imports.file_integrity import check_audio_integrity +from core.imports.file_integrity import check_audio_integrity, resolve_duration_tolerance from core.imports.filename import extract_track_number_from_filename from core.imports.guards import check_flac_bit_depth, move_to_quarantine +from core.imports.quarantine import entry_id_from_quarantined_filename from core.imports.side_effects import ( emit_track_downloaded, record_download_provenance, @@ -79,6 +80,26 @@ __all__ = [ ] +def _should_skip_quarantine_check(context: dict, check_name: str) -> bool: + bypass = context.get('_skip_quarantine_check') + if bypass == 'all': + return True + if isinstance(bypass, (list, tuple, set)): + return 'all' in bypass or check_name in bypass + return bypass == check_name + + +def _mark_task_quarantined(context: dict, quarantine_path: str | None) -> None: + if not quarantine_path: + return + task_id = context.get('task_id') + if not task_id: + return + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['quarantine_entry_id'] = entry_id_from_quarantined_filename(quarantine_path) + + def build_import_pipeline_runtime( *, automation_engine: Any | None = None, @@ -155,11 +176,29 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta except Exception: _expected_duration_ms = None - try: - integrity = check_audio_integrity(file_path, _expected_duration_ms) - except Exception as integrity_error: - logger.error(f"[Integrity] Check raised unexpectedly (continuing): {integrity_error}") + # User-configurable tolerance override. None = use built-in + # auto-scaled defaults (3s normal / 5s for tracks >10min). Set + # higher (e.g. 10) when matched files routinely drift from the + # source's reported duration (live recordings, alternate + # masterings, etc). + _duration_tolerance_override = resolve_duration_tolerance( + config_manager.get('post_processing.duration_tolerance_seconds', 0) + ) + # User-approved quarantine restores can bypass quarantine gates + # for this one post-processing pass. + if _should_skip_quarantine_check(context, 'integrity'): + logger.info(f"[Integrity] Skipped (user approval) for {_basename}") integrity = None + else: + try: + integrity = check_audio_integrity( + file_path, + _expected_duration_ms, + length_tolerance_s=_duration_tolerance_override, + ) + except Exception as integrity_error: + logger.error(f"[Integrity] Check raised unexpectedly (continuing): {integrity_error}") + integrity = None if integrity is not None and not integrity.ok: logger.error(f"[Integrity] Rejected {_basename}: {integrity.reason}") @@ -171,7 +210,9 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta context, f"Integrity check failed: {integrity.reason}", automation_engine, + trigger='integrity', ) + _mark_task_quarantined(context, quarantine_path) logger.error(f"File quarantined due to integrity failure: {quarantine_path}") except Exception as quarantine_error: logger.error(f"Quarantine failed ({quarantine_error}), deleting broken file: {file_path}") @@ -206,7 +247,9 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta f"drift={integrity.checks.get('length_drift_s', 'n/a')})" ) - _skip_acoustid = False + _skip_acoustid = _should_skip_quarantine_check(context, 'acoustid') + if _skip_acoustid: + logger.info(f"[AcoustID] Skipped (user approval) for {_basename}") try: from core.acoustid_verification import AcoustIDVerification, VerificationResult @@ -248,7 +291,9 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta context, verification_msg, automation_engine, + trigger='acoustid', ) + _mark_task_quarantined(context, quarantine_path) logger.error(f"File quarantined due to verification failure: {quarantine_path}") except Exception as quarantine_error: logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}") @@ -420,7 +465,10 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta if context['_audio_quality']: logger.info(f"Audio quality detected: {context['_audio_quality']}") - rejection_reason = check_flac_bit_depth(file_path, context) + _skip_bit_depth = _should_skip_quarantine_check(context, 'bit_depth') + rejection_reason = None if _skip_bit_depth else check_flac_bit_depth(file_path, context) + if _skip_bit_depth: + logger.info(f"[BitDepth] Skipped (user approval) for {_basename}") if rejection_reason: try: quarantine_path = move_to_quarantine( @@ -428,7 +476,9 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta context, rejection_reason, automation_engine, + trigger='bit_depth', ) + _mark_task_quarantined(context, quarantine_path) logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") except Exception as quarantine_error: logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") @@ -548,7 +598,10 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta if context['_audio_quality']: logger.info(f"Audio quality detected: {context['_audio_quality']}") - rejection_reason = check_flac_bit_depth(file_path, context) + _skip_bit_depth = _should_skip_quarantine_check(context, 'bit_depth') + rejection_reason = None if _skip_bit_depth else check_flac_bit_depth(file_path, context) + if _skip_bit_depth: + logger.info(f"[BitDepth] Skipped (user approval) for {_basename}") if rejection_reason: try: quarantine_path = move_to_quarantine( @@ -556,7 +609,9 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta context, rejection_reason, automation_engine, + trigger='bit_depth', ) + _mark_task_quarantined(context, quarantine_path) logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") except Exception as quarantine_error: logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") diff --git a/core/imports/quarantine.py b/core/imports/quarantine.py new file mode 100644 index 00000000..43ed5191 --- /dev/null +++ b/core/imports/quarantine.py @@ -0,0 +1,405 @@ +"""Quarantine entry management — pure helpers for list/delete/approve/recover. + +Quarantined files live in `/ss_quarantine/` as +`_..quarantined` paired with a JSON sidecar +`_.json` written by `core.imports.guards.move_to_quarantine`. + +This module provides the read/write/restore primitives. Web routes are +thin glue around these. Pipeline re-run on approval is the caller's +job (we hand back `(file_path, context, bypass_check)`). +""" + +from __future__ import annotations + +import json +import os +import shutil +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from utils.logging_config import get_logger + +logger = get_logger("imports.quarantine") + + +_QUARANTINE_SUFFIX = ".quarantined" + + +# JSON-serializable scalar predicate. dict / list values get walked +# recursively; anything else is dropped during sidecar serialization. +_SAFE_SCALARS = (str, int, float, bool, type(None)) + + +def serialize_quarantine_context(context: Any) -> Dict[str, Any]: + """Walk a context dict and emit a JSON-safe copy. + + Drops non-serializable values (sets, custom objects, callables, + open file handles, etc) silently — sidecar must round-trip through + `json.dump` / `json.load` without raising. Lists are walked element + by element; dicts are walked recursively. Anything that isn't a + scalar / dict / list is converted to a string fallback so caller + still sees *something* (rather than a silent drop) but won't break + the JSON write. + """ + if not isinstance(context, dict): + return {} + return _coerce_dict(context) + + +def _coerce_value(value: Any) -> Any: + if isinstance(value, _SAFE_SCALARS): + return value + if isinstance(value, dict): + return _coerce_dict(value) + if isinstance(value, (list, tuple)): + return [_coerce_value(v) for v in value] + if isinstance(value, set): + return [_coerce_value(v) for v in value] + # Fallback — preserve via str() so caller sees the value's shape + # without breaking JSON serialization. + try: + return str(value) + except Exception: + return None + + +def _coerce_dict(d: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + for key, value in d.items(): + if not isinstance(key, str): + try: + key = str(key) + except Exception: + continue + out[key] = _coerce_value(value) + return out + + +def _entry_id_from_filename(quarantined_filename: str) -> str: + """Derive a stable entry id from the quarantined filename. + + Strip the `.quarantined` suffix; strip the original file extension; + return the bare `_` stem. Sidecar uses the + same stem with a `.json` extension, so the id pairs both sides. + """ + base = quarantined_filename + if base.endswith(_QUARANTINE_SUFFIX): + base = base[: -len(_QUARANTINE_SUFFIX)] + return Path(base).stem + + +def entry_id_from_quarantined_filename(quarantined_filename: str) -> str: + """Derive a quarantine entry id from a quarantined filename or path.""" + return _entry_id_from_filename(os.path.basename(quarantined_filename)) + + +def get_quarantined_source_keys(quarantine_dir: str) -> set: + """Return a set of ``(username, filename)`` tuples for every Soulseek + source that has been quarantined. + + Used to gate the Soulseek candidate filter against re-picking the + exact same upload that already failed post-download verification. + Issue #652 — without this gate, the auto-wishlist processor's + candidate ranking is deterministic, so the same `(uploader, file)` + keeps winning the quality picker, downloading, quarantining, and + re-queueing in an infinite loop. Users wake up to hundreds of + duplicate `.quarantined` files for the same source URL. + + The keys come from the sidecar JSON's + ``context.original_search_result`` field which `move_to_quarantine` + persists from the originating SearchResult. Sidecars missing either + field (legacy thin sidecars written pre-Feb 2026, or orphaned + files) are skipped silently — they can't gate anything anyway. + + Returns an empty set when the directory doesn't exist or has no + parseable sidecars. Never raises; filesystem / JSON errors are + swallowed at debug level so a corrupt sidecar can't block the + download pipeline. + """ + keys: set = set() + if not quarantine_dir or not os.path.isdir(quarantine_dir): + return keys + + try: + names = os.listdir(quarantine_dir) + except OSError as exc: + logger.debug("get_quarantined_source_keys: listdir failed: %s", exc) + return keys + + for name in names: + if not name.endswith('.json'): + continue + sidecar_path = os.path.join(quarantine_dir, name) + try: + with open(sidecar_path, encoding='utf-8') as f: + sidecar = json.load(f) + except Exception as exc: + logger.debug("get_quarantined_source_keys: sidecar read failed for %s: %s", name, exc) + continue + if not isinstance(sidecar, dict): + continue + ctx = sidecar.get('context') + if not isinstance(ctx, dict): + continue + osr = ctx.get('original_search_result') + if not isinstance(osr, dict): + continue + username = osr.get('username') or '' + filename = osr.get('filename') or '' + if username and filename: + keys.add((str(username), str(filename))) + + return keys + + +def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]: + """Enumerate quarantined files paired with their sidecars. + + Returns one dict per `.quarantined` file with: id, filename, + original_filename (from sidecar), reason, expected_track, + expected_artist, timestamp, size_bytes, has_full_context (True + when the sidecar carries a `context` field — required for one-click + Approve), trigger (which check fired: integrity / acoustid / + bit_depth / unknown). + + Orphaned `.quarantined` files (no sidecar) still surface — caller + can delete them. Orphaned sidecars (no file) are skipped silently. + Sorted newest-first by timestamp prefix. + """ + entries: List[Dict[str, Any]] = [] + if not os.path.isdir(quarantine_dir): + return entries + + for name in os.listdir(quarantine_dir): + if not name.endswith(_QUARANTINE_SUFFIX): + continue + full_path = os.path.join(quarantine_dir, name) + if not os.path.isfile(full_path): + continue + + entry_id = _entry_id_from_filename(name) + sidecar_path = os.path.join(quarantine_dir, f"{entry_id}.json") + sidecar: Dict[str, Any] = {} + if os.path.isfile(sidecar_path): + try: + with open(sidecar_path, encoding="utf-8") as f: + loaded = json.load(f) + if isinstance(loaded, dict): + sidecar = loaded + except Exception as exc: + logger.debug("sidecar read failed for %s: %s", entry_id, exc) + + try: + size_bytes = os.path.getsize(full_path) + except OSError: + size_bytes = 0 + + # Issue #608 follow-up (AfonsoG6): surface the source username + # + filename that was originally downloaded, so the user can see + # at a glance which uploader the bad file came from. Lives + # under `context.original_search_result` when full context is + # persisted; absent on legacy thin sidecars. + ctx = sidecar.get("context") if isinstance(sidecar.get("context"), dict) else {} + osr = ctx.get("original_search_result") if isinstance(ctx.get("original_search_result"), dict) else {} + source_username = osr.get("username", "") if isinstance(osr, dict) else "" + source_filename = osr.get("filename", "") if isinstance(osr, dict) else "" + + entries.append( + { + "id": entry_id, + "filename": name, + "original_filename": sidecar.get("original_filename", name), + "reason": sidecar.get("quarantine_reason", "Unknown reason"), + "expected_track": sidecar.get("expected_track", ""), + "expected_artist": sidecar.get("expected_artist", ""), + "timestamp": sidecar.get("timestamp", ""), + "size_bytes": size_bytes, + "has_full_context": isinstance(sidecar.get("context"), dict), + "trigger": sidecar.get("trigger", "unknown"), + "source_username": source_username, + "source_filename": source_filename, + } + ) + + entries.sort(key=lambda e: e["id"], reverse=True) + return entries + + +def _resolve_entry_paths(quarantine_dir: str, entry_id: str) -> Tuple[Optional[str], Optional[str]]: + """Locate the `.quarantined` file + JSON sidecar for an entry id. + + Returns (file_path, sidecar_path), either may be None if missing. + """ + if not os.path.isdir(quarantine_dir) or not entry_id: + return None, None + file_path: Optional[str] = None + for name in os.listdir(quarantine_dir): + if not name.endswith(_QUARANTINE_SUFFIX): + continue + if _entry_id_from_filename(name) == entry_id: + file_path = os.path.join(quarantine_dir, name) + break + sidecar_path = os.path.join(quarantine_dir, f"{entry_id}.json") + if not os.path.isfile(sidecar_path): + sidecar_path = None + return file_path, sidecar_path + + +def delete_quarantine_entry(quarantine_dir: str, entry_id: str) -> bool: + """Delete the quarantined file + sidecar for the given entry id. + + Returns True if at least one of the two was removed. False when + neither existed (entry already gone). + """ + file_path, sidecar_path = _resolve_entry_paths(quarantine_dir, entry_id) + removed = False + if file_path and os.path.isfile(file_path): + try: + os.remove(file_path) + removed = True + except OSError as exc: + logger.error("Failed to delete quarantine file %s: %s", file_path, exc) + if sidecar_path and os.path.isfile(sidecar_path): + try: + os.remove(sidecar_path) + removed = True + except OSError as exc: + logger.error("Failed to delete quarantine sidecar %s: %s", sidecar_path, exc) + return removed + + +def _restore_filename(quarantined_filename: str, sidecar_original: Optional[str] = None) -> str: + """Resolve the filename to restore. + + Sidecar's `original_filename` wins when provided — it's the + canonical record of what the file was named before quarantine. + Otherwise parse the `_..quarantined` + convention written by `move_to_quarantine`, dropping the timestamp + prefix and `.quarantined` suffix. Final fallback returns the + quarantined filename minus the suffix unchanged. + """ + if sidecar_original: + return sidecar_original + base = quarantined_filename + if base.endswith(_QUARANTINE_SUFFIX): + base = base[: -len(_QUARANTINE_SUFFIX)] + parts = base.split("_", 2) + if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit(): + return parts[2] + return base + + +def approve_quarantine_entry( + quarantine_dir: str, + entry_id: str, + restore_dir: str, +) -> Optional[Tuple[str, Dict[str, Any], str]]: + """Restore a quarantined file for re-import via the post-process pipeline. + + Reads the sidecar's `context` + `trigger`, moves the file out of + quarantine to `restore_dir` (with the original filename + extension), + deletes the sidecar. + + Returns `(restored_file_path, context, trigger)` so the caller can + set the appropriate `_skip_quarantine_check` bypass flag and + dispatch the post-process pipeline. + + Returns None when: + - the entry doesn't exist + - the sidecar lacks a serialized `context` (legacy thin sidecar + — caller should fall back to `recover_to_staging` instead) + - the file move fails + """ + file_path, sidecar_path = _resolve_entry_paths(quarantine_dir, entry_id) + if not file_path or not sidecar_path: + logger.warning("approve: entry %s missing file or sidecar", entry_id) + return None + + try: + with open(sidecar_path, encoding="utf-8") as f: + sidecar = json.load(f) + except Exception as exc: + logger.error("approve: sidecar read failed for %s: %s", entry_id, exc) + return None + + context = sidecar.get("context") + if not isinstance(context, dict): + logger.info("approve: entry %s has thin sidecar (no context) — caller should recover-to-staging", entry_id) + return None + + trigger = str(sidecar.get("trigger", "unknown")) + + original_name = sidecar.get("original_filename") or _restore_filename(os.path.basename(file_path)) + os.makedirs(restore_dir, exist_ok=True) + restored_path = os.path.join(restore_dir, original_name) + restored_path = _ensure_unique_path(restored_path) + + try: + shutil.move(file_path, restored_path) + except OSError as exc: + logger.error("approve: failed to restore %s -> %s: %s", file_path, restored_path, exc) + return None + + try: + os.remove(sidecar_path) + except OSError as exc: + logger.warning("approve: failed to remove sidecar %s: %s", sidecar_path, exc) + + return restored_path, context, trigger + + +def recover_to_staging( + quarantine_dir: str, + staging_dir: str, + entry_id: str, +) -> Optional[str]: + """Move a quarantined file into Staging for manual import. + + Strips the timestamp prefix + `.quarantined` suffix, drops the file + into `staging_dir` so the user can finish via the existing Import + flow. Sidecar is removed. Used as the fallback path for legacy thin + sidecars (no embedded `context`) where one-click Approve is + impossible. + """ + file_path, sidecar_path = _resolve_entry_paths(quarantine_dir, entry_id) + if not file_path: + return None + + sidecar_original = None + if sidecar_path: + try: + with open(sidecar_path, encoding="utf-8") as f: + sidecar_original = json.load(f).get("original_filename") + except Exception as exc: + logger.debug("recover: sidecar read failed for %s: %s", entry_id, exc) + + restored_name = _restore_filename(os.path.basename(file_path), sidecar_original) + os.makedirs(staging_dir, exist_ok=True) + target = _ensure_unique_path(os.path.join(staging_dir, restored_name)) + + try: + shutil.move(file_path, target) + except OSError as exc: + logger.error("recover: failed to move %s -> %s: %s", file_path, target, exc) + return None + + if sidecar_path and os.path.isfile(sidecar_path): + try: + os.remove(sidecar_path) + except OSError as exc: + logger.warning("recover: failed to remove sidecar %s: %s", sidecar_path, exc) + + return target + + +def _ensure_unique_path(target: str) -> str: + """Append `_(2)`, `_(3)`, ... before the extension when target exists.""" + if not os.path.exists(target): + return target + base, ext = os.path.splitext(target) + counter = 2 + while True: + candidate = f"{base}_({counter}){ext}" + if not os.path.exists(candidate): + return candidate + counter += 1 diff --git a/core/imports/routes.py b/core/imports/routes.py new file mode 100644 index 00000000..edd38f08 --- /dev/null +++ b/core/imports/routes.py @@ -0,0 +1,522 @@ +"""Import/staging controller helpers for Flask-style endpoints.""" + +from __future__ import annotations + +import os +import uuid +from concurrent.futures import as_completed +from dataclasses import dataclass +from typing import Any, Callable, Dict + +from core.imports.album import build_album_import_context, build_album_import_match_payload, resolve_album_artist_context +from core.imports.context import get_import_context_artist, get_import_track_info, normalize_import_context +from core.imports.filename import parse_filename_metadata +from core.imports.staging import ( + AUDIO_EXTENSIONS, + get_import_suggestions_cache, + get_primary_source as _get_primary_source, + get_staging_path as _get_staging_path, + read_staging_file_metadata as _read_staging_file_metadata, + refresh_import_suggestions_cache as _refresh_import_suggestions_cache, + search_import_albums as _search_import_albums, + search_import_tracks as _search_import_tracks, +) +from utils.logging_config import get_logger + + +module_logger = get_logger("imports.routes") + + +def _default_read_tags(file_path: str): + from mutagen import File as MutagenFile + + return MutagenFile(file_path, easy=True) + + +def _get_single_track_import_context(*args, **kwargs): + from core.imports.resolution import get_single_track_import_context + + return get_single_track_import_context(*args, **kwargs) + + +@dataclass +class ImportRouteRuntime: + """Dependencies needed to service import/staging HTTP endpoints.""" + + get_staging_path: Callable[[], str] = _get_staging_path + read_staging_file_metadata: Callable[[str, str], Dict[str, Any]] = _read_staging_file_metadata + read_tags: Callable[[str], Any] = _default_read_tags + get_primary_source: Callable[[], str] = _get_primary_source + search_import_albums: Callable[..., list] = _search_import_albums + search_import_tracks: Callable[..., list] = _search_import_tracks + build_album_import_match_payload: Callable[..., Dict[str, Any]] = build_album_import_match_payload + resolve_album_artist_context: Callable[..., Any] = resolve_album_artist_context + build_album_import_context: Callable[..., Dict[str, Any]] = build_album_import_context + get_single_track_import_context: Callable[..., Dict[str, Any]] = _get_single_track_import_context + parse_filename_metadata: Callable[[str], Dict[str, Any]] = parse_filename_metadata + normalize_import_context: Callable[[Dict[str, Any]], Dict[str, Any]] = normalize_import_context + get_import_context_artist: Callable[[Dict[str, Any]], Dict[str, Any]] = get_import_context_artist + get_import_track_info: Callable[[Dict[str, Any]], Dict[str, Any]] = get_import_track_info + process_single_import_file: Callable[["ImportRouteRuntime", Dict[str, Any]], tuple[str, str]] | None = None + post_process_matched_download: Callable[[str, Dict[str, Any], str], Any] | None = None + add_activity_item: Callable[[Any, Any, Any, Any], Any] | None = None + refresh_import_suggestions_cache: Callable[[], Any] = _refresh_import_suggestions_cache + automation_engine: Any = None + hydrabase_worker: Any = None + dev_mode_enabled: bool = False + import_singles_executor: Any = None + logger: Any = module_logger + + +def staging_files(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]: + """Scan the staging folder and return audio files with tag metadata.""" + try: + staging_path = runtime.get_staging_path() + os.makedirs(staging_path, exist_ok=True) + + files = [] + for root, _dirs, filenames in os.walk(staging_path): + for fname in filenames: + ext = os.path.splitext(fname)[1].lower() + if ext not in AUDIO_EXTENSIONS: + continue + full_path = os.path.join(root, fname) + rel_path = os.path.relpath(full_path, staging_path) + + meta = runtime.read_staging_file_metadata(full_path, rel_path) + + files.append( + { + "filename": fname, + "rel_path": rel_path, + "full_path": full_path, + "title": meta["title"], + "artist": meta["albumartist"] or meta["artist"] or "Unknown Artist", + "album": meta["album"], + "track_number": meta["track_number"], + "disc_number": meta["disc_number"], + "extension": ext, + } + ) + + files.sort(key=lambda f: f["filename"].lower()) + return {"success": True, "files": files, "staging_path": staging_path}, 200 + except Exception as exc: + runtime.logger.error("Error scanning staging files: %s", exc) + return {"success": False, "error": str(exc)}, 500 + + +def staging_groups(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]: + """Auto-detect album groups from staging files based on their tags.""" + try: + staging_path = runtime.get_staging_path() + if not os.path.isdir(staging_path): + return {"success": True, "groups": []}, 200 + + album_groups = {} + for root, _dirs, filenames in os.walk(staging_path): + for fname in filenames: + ext = os.path.splitext(fname)[1].lower() + if ext not in AUDIO_EXTENSIONS: + continue + full_path = os.path.join(root, fname) + rel_path = os.path.relpath(full_path, staging_path) + + meta = runtime.read_staging_file_metadata(full_path, rel_path) + album = meta["album"] + artist = meta["albumartist"] or meta["artist"] + if not album or not artist: + continue + + key = (album.lower().strip(), artist.lower().strip()) + if key not in album_groups: + album_groups[key] = {"album": album.strip(), "artist": artist.strip(), "files": []} + album_groups[key]["files"].append( + { + "filename": fname, + "full_path": full_path, + "title": meta["title"], + "track_number": meta["track_number"], + } + ) + + groups = [] + for group in album_groups.values(): + if len(group["files"]) >= 2: + group["files"].sort(key=lambda f: f.get("track_number") or 999) + groups.append( + { + "album": group["album"], + "artist": group["artist"], + "file_count": len(group["files"]), + "files": group["files"], + "file_paths": [f["full_path"] for f in group["files"]], + } + ) + + groups.sort(key=lambda g: g["file_count"], reverse=True) + return {"success": True, "groups": groups}, 200 + except Exception as exc: + runtime.logger.error("Error building staging groups: %s", exc) + return {"success": False, "error": str(exc)}, 500 + + +def staging_hints(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]: + """Extract album search hints from staging folder tags and folder names.""" + try: + staging_path = runtime.get_staging_path() + if not os.path.isdir(staging_path): + return {"success": True, "hints": []}, 200 + + tag_albums = {} + folder_hints = {} + for root, _dirs, filenames in os.walk(staging_path): + audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] + if not audio_files: + continue + + rel_dir = os.path.relpath(root, staging_path) + if rel_dir != ".": + top_folder = rel_dir.split(os.sep)[0] + folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) + + for fname in audio_files: + full_path = os.path.join(root, fname) + try: + tags = runtime.read_tags(full_path) + if tags: + album = (tags.get("album") or [None])[0] + artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0] + if album: + key = (album.strip(), (artist or "").strip()) + tag_albums[key] = tag_albums.get(key, 0) + 1 + except Exception as exc: + runtime.logger.debug("tag read failed: %s", exc) + + queries = [] + seen_queries_lower = set() + + for (album, artist), _count in sorted(tag_albums.items(), key=lambda x: -x[1]): + query = f"{album} {artist}".strip() if artist else album + if query.lower() not in seen_queries_lower: + seen_queries_lower.add(query.lower()) + queries.append(query) + + for folder, _count in sorted(folder_hints.items(), key=lambda x: -x[1]): + query = folder.replace("_", " ") + if query.lower() not in seen_queries_lower: + seen_queries_lower.add(query.lower()) + queries.append(query) + + return {"success": True, "hints": queries[:5]}, 200 + except Exception as exc: + runtime.logger.error("Error getting staging hints: %s", exc) + return {"success": False, "error": str(exc)}, 500 + + +def staging_suggestions() -> tuple[Dict[str, Any], int]: + """Return cached import suggestions and readiness state.""" + cache = get_import_suggestions_cache() + return { + "success": True, + "suggestions": cache["suggestions"], + "ready": cache["built"], + "primary_source": _get_primary_source(), + }, 200 + + +def search_albums(runtime: ImportRouteRuntime, query: str, limit: int = 12) -> tuple[Dict[str, Any], int]: + """Search albums for manual import using the active metadata provider.""" + try: + query = (query or "").strip() + if not query: + return {"success": False, "error": "Missing query parameter"}, 400 + + limit = min(int(limit), 50) + primary_source = runtime.get_primary_source() + if primary_source == "hydrabase" and runtime.hydrabase_worker and runtime.dev_mode_enabled: + runtime.hydrabase_worker.enqueue(query, "albums") + + albums = runtime.search_import_albums(query, limit=limit) + return {"success": True, "albums": albums, "primary_source": primary_source}, 200 + except Exception as exc: + runtime.logger.error("Error searching albums for import: %s", exc) + return {"success": False, "error": str(exc)}, 500 + + +def album_match(runtime: ImportRouteRuntime, data: Dict[str, Any]) -> tuple[Dict[str, Any], int]: + """Match staging files to an album's tracklist.""" + try: + data = data or {} + album_id = data.get("album_id") + album_name = data.get("album_name", "") + album_artist = data.get("album_artist", "") + source = str(data.get("source") or "").strip().lower() + filter_file_paths = set(data.get("file_paths", [])) + if not album_id: + return {"success": False, "error": "Missing album_id"}, 400 + + if not source: + runtime.logger.warning( + "[Import Match] Missing 'source' on album_id=%s - lookup will " + "guess via primary-source priority chain. If this fires " + "consistently, a frontend caller is dropping source from " + "the match POST body.", + album_id, + ) + + payload = runtime.build_album_import_match_payload( + album_id, + album_name=album_name, + album_artist=album_artist, + file_paths=filter_file_paths, + source=source or None, + ) + return payload, 200 + except Exception as exc: + runtime.logger.error("Error matching album for import: %s", exc) + return {"success": False, "error": str(exc)}, 500 + + +def album_process(runtime: ImportRouteRuntime, data: Dict[str, Any]) -> tuple[Dict[str, Any], int]: + """Process matched album files through the post-processing pipeline.""" + try: + data = data or {} + album = data.get("album", {}) + matches = data.get("matches", []) + + if not album or not matches: + return {"success": False, "error": "Missing album or matches data"}, 400 + if runtime.post_process_matched_download is None: + return {"success": False, "error": "Import post-processing not available"}, 500 + + processed = 0 + errors = [] + album_name = album.get("name", album.get("album_name", "Unknown Album")) + artist_name = album.get("artist", album.get("artist_name", "Unknown Artist")) + album_id = album.get("id", album.get("album_id", "")) + source = str(album.get("source") or data.get("source") or "").strip().lower() + + total_discs = max( + ( + match.get("track", {}).get("disc_number", 1) + for match in matches + if match.get("track") + ), + default=1, + ) + artist_context = runtime.resolve_album_artist_context(album, source=source) + + for match in matches: + staging_file = match.get("staging_file") + track = match.get("track") or {} + if not staging_file or not track: + continue + + file_path = staging_file.get("full_path", "") + if not os.path.isfile(file_path): + errors.append(f"File not found: {staging_file.get('filename', '?')}") + continue + + track_name = track.get("name", "Unknown Track") + track_number = track.get("track_number", 1) + context_key = f"import_album_{album_id}_{track_number}_{uuid.uuid4().hex[:8]}" + context = runtime.build_album_import_context( + album, + track, + artist_context=artist_context, + total_discs=total_discs, + source=source, + ) + + try: + runtime.post_process_matched_download(context_key, context, file_path) + processed += 1 + runtime.logger.info("Import processed: %s. %s from %s", track_number, track_name, album_name) + except Exception as proc_err: + err_msg = f"{track_name}: {str(proc_err)}" + errors.append(err_msg) + runtime.logger.error("Import processing error: %s", err_msg) + + if runtime.add_activity_item: + runtime.add_activity_item("", "Album Imported", f"{album_name} by {artist_name} ({processed}/{len(matches)} tracks)", "Now") + + if processed > 0: + _emit_import_completed( + runtime, + track_count=processed, + album_name=album_name or "", + artist=artist_name or "", + playlist_name=f"Import: {album_name}" if album_name else "Import", + total_tracks=len(matches), + failed_tracks=len(errors), + log_label="album", + ) + runtime.refresh_import_suggestions_cache() + + return {"success": True, "processed": processed, "total": len(matches), "errors": errors}, 200 + except Exception as exc: + runtime.logger.error("Error processing album import: %s", exc) + return {"success": False, "error": str(exc)}, 500 + + +def search_tracks(runtime: ImportRouteRuntime, query: str, limit: int = 10) -> tuple[Dict[str, Any], int]: + """Search tracks for manual single import using metadata source priority.""" + try: + query = (query or "").strip() + if not query: + return {"success": False, "error": "Missing query parameter"}, 400 + + limit = min(int(limit), 30) + primary_source = runtime.get_primary_source() + if primary_source == "hydrabase" and runtime.hydrabase_worker and runtime.dev_mode_enabled: + runtime.hydrabase_worker.enqueue(query, "tracks") + + tracks = runtime.search_import_tracks(query, limit=limit) + return {"success": True, "tracks": tracks, "primary_source": primary_source}, 200 + except Exception as exc: + runtime.logger.error("Error searching tracks for import: %s", exc) + return {"success": False, "error": str(exc)}, 500 + + +def process_single_import_file(runtime: ImportRouteRuntime, file_info: Dict[str, Any]) -> tuple[str, str]: + """Validate, resolve metadata, and post-process one single import file.""" + file_path = file_info.get("full_path", "") + if not os.path.isfile(file_path): + return ("error", f"File not found: {file_info.get('filename', '?')}") + if runtime.post_process_matched_download is None: + return ("error", "Import post-processing not available") + + title = file_info.get("title", "") + artist = file_info.get("artist", "") + manual_match = file_info.get("manual_match") + if manual_match is not None and not isinstance(manual_match, dict): + manual_match = None + + manual_match_source = "" + manual_match_id = None + if manual_match: + manual_match_source = str(manual_match.get("source") or "").strip().lower() + manual_match_id = str(manual_match.get("id") or "").strip() + if not manual_match_id or not manual_match_source: + return ("error", f"Malformed manual match for file: {file_info.get('filename', '?')}") + + if not title and not manual_match: + parsed = runtime.parse_filename_metadata(file_info.get("filename", "")) + title = parsed.get("title") or os.path.splitext(file_info.get("filename", "Unknown"))[0] + if not artist: + artist = parsed.get("artist", "") + + try: + resolved = runtime.get_single_track_import_context( + title, + artist, + override_id=manual_match_id, + override_source=manual_match_source, + ) + context = runtime.normalize_import_context(resolved["context"]) + artist_data = runtime.get_import_context_artist(context) + track_data = runtime.get_import_track_info(context) + final_title = track_data.get("name", title) + final_artist = artist_data.get("name", artist) + + context_key = f"import_single_{uuid.uuid4().hex[:8]}" + runtime.post_process_matched_download(context_key, context, file_path) + runtime.logger.info( + "Import single processed: %s by %s (source=%s)", + final_title, + final_artist, + resolved.get("source") or "local", + ) + return ("ok", final_title) + except Exception as proc_err: + err_msg = f"{title}: {str(proc_err)}" + runtime.logger.error("Import single processing error: %s", err_msg) + return ("error", err_msg) + + +def singles_process(runtime: ImportRouteRuntime, files: list[Dict[str, Any]]) -> tuple[Dict[str, Any], int]: + """Process individual staging files as singles through the import pipeline.""" + try: + files = files or [] + if not files: + return {"success": False, "error": "No files provided"}, 400 + if runtime.import_singles_executor is None: + return {"success": False, "error": "Import executor not available"}, 500 + + processed = 0 + errors = [] + process_file = runtime.process_single_import_file or process_single_import_file + future_to_filename = { + runtime.import_singles_executor.submit(process_file, runtime, file_info): + file_info.get("filename", "?") + for file_info in files + } + + for future in as_completed(future_to_filename): + try: + outcome, payload = future.result() + except Exception as worker_err: + errors.append(f"{future_to_filename[future]}: worker crashed: {worker_err}") + continue + if outcome == "ok": + processed += 1 + else: + errors.append(payload) + + if runtime.add_activity_item: + runtime.add_activity_item("", "Singles Imported", f"{processed}/{len(files)} tracks processed", "Now") + + if processed > 0: + _emit_import_completed( + runtime, + track_count=processed, + album_name="", + artist="Various", + playlist_name="Import: Singles", + total_tracks=len(files), + failed_tracks=len(errors), + log_label="singles", + ) + runtime.refresh_import_suggestions_cache() + + return {"success": True, "processed": processed, "total": len(files), "errors": errors}, 200 + except Exception as exc: + runtime.logger.error("Error processing singles import: %s", exc) + return {"success": False, "error": str(exc)}, 500 + + +def _emit_import_completed( + runtime: ImportRouteRuntime, + *, + track_count: int, + album_name: str, + artist: str, + playlist_name: str, + total_tracks: int, + failed_tracks: int, + log_label: str, +) -> None: + # Keep import automation on the same chain as download batches: + # batch_complete -> auto-scan -> library_scan_completed -> auto-update DB. + try: + if runtime.automation_engine: + runtime.automation_engine.emit( + "import_completed", + { + "track_count": str(track_count), + "album_name": album_name, + "artist": artist, + }, + ) + runtime.automation_engine.emit( + "batch_complete", + { + "playlist_name": playlist_name, + "total_tracks": str(total_tracks), + "completed_tracks": str(track_count), + "failed_tracks": str(failed_tracks), + }, + ) + except Exception as exc: + runtime.logger.debug("%s import automation emit failed: %s", log_label, exc) diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index 0868d6a2..6513aabf 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -198,6 +198,10 @@ def record_library_history_download(context: Dict[str, Any]) -> None: "deezer_dl": "Deezer", "lidarr": "Lidarr", "soundcloud": "SoundCloud", + "amazon": "Amazon", + "staging": "Staging", + "torrent": "Torrent", + "usenet": "Usenet", # Auto-import isn't a download source, but flows through the # same post-process pipeline (file lands → record provenance # + history → write to library DB). Tagging it as "Auto-Import" @@ -235,7 +239,7 @@ def record_library_history_download(context: Dict[str, Any]) -> None: source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "") source_track_title = search_result.get("title", "") or search_result.get("name", "") source_artist = search_result.get("artist", "") - if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr", "soundcloud"): + if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr", "soundcloud", "amazon"): stream_id = source_filename.split("||")[0] if stream_id and not source_track_id: source_track_id = stream_id @@ -276,6 +280,7 @@ def record_download_provenance(context: Dict[str, Any]) -> None: "deezer_dl": "deezer", "lidarr": "lidarr", "soundcloud": "soundcloud", + "amazon": "amazon", # Auto-import: surfaced in provenance so the redownload modal # can tell the user "this came from staging on " instead # of falsely listing soulseek as the source. The underlying @@ -283,6 +288,16 @@ def record_download_provenance(context: Dict[str, Any]) -> None: # separately via the source-aware ID columns on the tracks # row itself. "auto_import": "auto_import", + # Generic staging-match (user dropped files manually OR a + # source we don't have a more specific label for). Better + # than defaulting to 'soulseek' which would falsely tag the + # provenance. + "staging": "staging", + # Torrent / usenet album-bundle flow — the staging matcher + # overrides 'staging' with the bundle source so the history + # shows where the files actually came from. + "torrent": "torrent", + "usenet": "usenet", }.get(username, "soulseek") ti = context.get("track_info") or context.get("search_result") or {} diff --git a/core/library/manual_library_match.py b/core/library/manual_library_match.py new file mode 100644 index 00000000..132ad729 --- /dev/null +++ b/core/library/manual_library_match.py @@ -0,0 +1,283 @@ +"""Manual library match service. + +Lets users explicitly link a source track (wishlist/sync-history candidate) to +an existing library track so SoulSync stops trying to re-download it. +""" + +from __future__ import annotations + +import json +from typing import Any, Optional + +from utils.logging_config import get_logger + +logger = get_logger("library.manual_library_match") + + +def save_match( + db, + profile_id: int, + source: str, + source_track_id: str, + library_track_id: int, + **meta, +) -> bool: + """Save (insert or replace) a manual match.""" + return db.save_manual_library_match( + profile_id, source, source_track_id, library_track_id, **meta + ) + + +def get_match( + db, + profile_id: int, + source: str, + source_track_id: str, + server_source: str = "", +) -> Optional[dict]: + """Return match row dict or None if not found.""" + getter = getattr(db, "get_manual_library_match", None) + if getter is None: + return None + return getter(profile_id, source, source_track_id, server_source) + + +def _first_artist_name(track: dict[str, Any]) -> str: + artists = track.get("artists") or [] + if isinstance(artists, list) and artists: + first = artists[0] + if isinstance(first, dict): + return (first.get("name") or "").strip() + return str(first).strip() + return (track.get("artist") or track.get("artist_name") or "").strip() + + +def _track_source_candidates(track: dict[str, Any], default_source: str = "") -> list[str]: + candidates = [ + track.get("provider"), + track.get("source"), + default_source, + "spotify", + ] + out = [] + for source in candidates: + source = (source or "").strip() + if source and source not in out: + out.append(source) + return out + + +def _track_id_candidates(track: dict[str, Any]) -> list[str]: + candidates = [ + track.get("source_track_id"), + track.get("spotify_track_id"), + track.get("track_id"), + track.get("id"), + track.get("musicbrainz_recording_id"), + track.get("deezer_id") or track.get("deezer_track_id"), + track.get("itunes_track_id"), + track.get("tidal_id") or track.get("tidal_track_id"), + track.get("qobuz_id") or track.get("qobuz_track_id"), + track.get("amazon_id") or track.get("amazon_track_id"), + ] + out = [] + for value in candidates: + value = str(value).strip() if value is not None else "" + if value and value not in out: + out.append(value) + return out + + +def get_match_for_track( + db, + profile_id: int, + track: dict[str, Any], + *, + default_source: str = "", + server_source: str = "", +) -> Optional[dict]: + """Return a manual match for a wishlist/sync track. + + Exact source+ID matches are preferred, but source labels can legitimately + change between UI surfaces (for example ``mirrored`` in sync history versus + ``wishlist`` in the wishlist batch). Fall back to track ID and finally + title/artist so saved manual matches are honored consistently. + """ + if not isinstance(track, dict): + return None + + sources = _track_source_candidates(track, default_source) + track_ids = _track_id_candidates(track) + for track_id in track_ids: + for source in sources: + match = get_match(db, profile_id, source, track_id, server_source) + if match: + return match + + id_getter = getattr(db, "find_manual_library_match_by_source_track_id", None) + if id_getter is not None: + for track_id in track_ids: + match = id_getter(profile_id, track_id, server_source) + if match: + return match + + title = (track.get("name") or track.get("title") or track.get("track_name") or "").strip() + artist = _first_artist_name(track) + metadata_getter = getattr(db, "find_manual_library_match_by_metadata", None) + if metadata_getter is not None and title and artist: + return metadata_getter(profile_id, title, artist, server_source) + return None + + +def delete_match(db, match_id: int, profile_id: int) -> bool: + """Delete match by PK id, scoped to profile.""" + return db.delete_manual_library_match(match_id, profile_id) + + +def list_matches(db, profile_id: int, limit: int = 100) -> list[dict]: + """Return all matches for profile, most-recently-updated first.""" + rows = db.list_manual_library_matches(profile_id, limit) + return [_enrich_match(row, db) for row in rows] + + +def search_source_candidates(db, query: str, profile_id: int, limit: int = 15) -> list[dict]: + """Search wishlist + sync history for source track candidates matching query.""" + if not query or not query.strip(): + return [] + + q = query.strip() + like = f"%{q}%" + results: dict[tuple, dict] = {} + + # 1) Wishlist tracks + try: + with db._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT + json_extract(spotify_data, '$.id') AS track_id, + json_extract(spotify_data, '$.name') AS title, + json_extract(spotify_data, '$.artists[0].name') AS artist, + json_extract(spotify_data, '$.album.name') AS album, + date_added AS added_at + FROM wishlist_tracks + WHERE profile_id = ? + AND ( + json_extract(spotify_data, '$.name') LIKE ? + OR json_extract(spotify_data, '$.artists[0].name') LIKE ? + ) + ORDER BY date_added DESC + LIMIT ? + """, (profile_id, like, like, limit * 2)) + for row in cursor.fetchall(): + r = dict(row) + if not r.get("track_id"): + continue + key = ("spotify", r["track_id"]) + if key not in results: + results[key] = { + "source": "spotify", + "source_track_id": r["track_id"], + "title": r["title"] or "", + "artist": r["artist"] or "", + "album": r["album"] or "", + "context": "Wishlist", + "added_at": r["added_at"] or "", + } + except Exception as exc: + logger.debug("source_candidates wishlist query failed: %s", exc) + + # 2) Sync history — scan tracks_json blobs + try: + with db._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT playlist_name, source, tracks_json, started_at + FROM sync_history + ORDER BY started_at DESC + LIMIT 50 + """) + for row in cursor.fetchall(): + sh = dict(row) + try: + tracks = json.loads(sh["tracks_json"] or "[]") + except Exception: + continue + for t in tracks: + title = t.get("name", "") + artist = "" + artists = t.get("artists", []) + if artists: + first = artists[0] + artist = first.get("name", "") if isinstance(first, dict) else str(first) + if q.lower() not in title.lower() and q.lower() not in artist.lower(): + continue + src = sh["source"] or "spotify" + tid = t.get("id") or t.get("spotify_track_id") or "" + if not tid: + continue + key = (src, tid) + if key not in results: + album = "" + alb = t.get("album") + if isinstance(alb, dict): + album = alb.get("name", "") + elif isinstance(alb, str): + album = alb + results[key] = { + "source": src, + "source_track_id": tid, + "title": title, + "artist": artist, + "album": album, + "context": sh["playlist_name"] or "", + "added_at": sh["started_at"] or "", + } + if len(results) >= limit * 3: + break + except Exception as exc: + logger.debug("source_candidates sync_history query failed: %s", exc) + + # Sort by recency and cap + sorted_results = sorted(results.values(), key=lambda r: r.get("added_at", ""), reverse=True) + return sorted_results[:limit] + + +def search_library_candidates(db, query: str, limit: int = 15) -> list[dict[str, Any]]: + """Search library tracks using the existing api_search_tracks method.""" + if not query or not query.strip(): + return [] + q = query.strip() + # Pass the full query as title (covers most single-field searches). + # Also try as artist in parallel and merge, deduped by track id. + title_rows = db.api_search_tracks(title=q, limit=limit) + artist_rows = db.api_search_tracks(artist=q, limit=limit) + seen: set[int] = set() + merged = [] + for row in title_rows + artist_rows: + rid = row.get('id') + if rid not in seen: + seen.add(rid) + merged.append(row) + if len(merged) >= limit: + break + return merged + + +def _enrich_match(match_row: dict, db) -> dict: + """Add library track details to a match row.""" + out = dict(match_row) + lib_id = match_row.get("library_track_id") + if lib_id: + try: + tracks = db.api_get_tracks_by_ids([lib_id]) + if tracks: + t = tracks[0] + out["library_title"] = t.get("title", "") + out["library_artist"] = t.get("artist_name", "") + out["library_album"] = t.get("album_title", "") + out["library_file_path"] = t.get("file_path", "") + out["library_bitrate"] = t.get("bitrate") + except Exception as exc: + logger.debug("enrich_match track lookup failed for id=%s: %s", lib_id, exc) + return out diff --git a/core/library/missing_track_import.py b/core/library/missing_track_import.py new file mode 100644 index 00000000..7b322be4 --- /dev/null +++ b/core/library/missing_track_import.py @@ -0,0 +1,605 @@ +"""Import an existing library file into a missing album slot. + +This module keeps the "I Have This" behavior out of the Flask route layer: +copy the selected source file, post-process it with target album metadata, +inherit album identity tags from target siblings, and write the real DB row. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import logging +import os +import shutil +import uuid +from typing import Any, Callable, Dict, Optional + +from core.library_reorganize import _build_post_process_context + + +logger = logging.getLogger("soulsync.library.missing_track_import") + + +class MissingTrackImportError(Exception): + """Expected import failure that should be surfaced to the API caller.""" + + def __init__(self, message: str, status_code: int = 400): + super().__init__(message) + self.status_code = status_code + + +@dataclass +class MissingTrackImportDeps: + database: Any + config_manager: Any + post_process_fn: Callable[[str, dict, str], Any] + resolve_library_file_path_fn: Callable[[Optional[str]], Optional[str]] + docker_resolve_path_fn: Callable[[str], str] + sync_tracks_to_server_fn: Optional[Callable[[list, str], Any]] = None + service_id_columns: Optional[Dict[str, Dict[str, str]]] = None + + +_ALBUM_IDENTITY_TAGS = { + "album", + "albumartist", + "album_artist", + "date", + "year", + "tracktotal", + "totaltracks", + "totaldiscs", + "musicbrainz_albumid", + "musicbrainz_albumartistid", + "musicbrainz_releasegroupid", + "barcode", + "catalognumber", + "originaldate", + "releasecountry", + "releasestatus", + "releasetype", + "media", + "script", + "copyright", + "spotify_album_id", + "deezer_album_id", + "tidal_album_id", + "qobuz_album_id", + "itunes_album_id", + "audiodb_album_id", +} + +_ID3_STANDARD_TAGS = { + "album": "TALB", + "albumartist": "TPE2", + "album_artist": "TPE2", + "date": "TDRC", + "year": "TDRC", +} + +_ID3_TXXX_DESCS = { + "musicbrainz_albumid": "MusicBrainz Album Id", + "musicbrainz_albumartistid": "MusicBrainz Album Artist Id", + "musicbrainz_releasegroupid": "MusicBrainz Release Group Id", + "barcode": "BARCODE", + "catalognumber": "CATALOGNUMBER", + "originaldate": "ORIGINALDATE", + "releasecountry": "RELEASECOUNTRY", + "releasestatus": "RELEASESTATUS", + "releasetype": "RELEASETYPE", + "media": "MEDIA", + "script": "SCRIPT", + "totaldiscs": "TOTALDISCS", + "tracktotal": "TOTALTRACKS", + "totaltracks": "TOTALTRACKS", + "spotify_album_id": "Spotify Album Id", + "deezer_album_id": "Deezer Album Id", + "tidal_album_id": "Tidal Album Id", + "qobuz_album_id": "Qobuz Album Id", + "itunes_album_id": "iTunes Album Id", + "audiodb_album_id": "AudioDB Album Id", +} + +_MP4_STANDARD_TAGS = { + "album": "\xa9alb", + "albumartist": "aART", + "album_artist": "aART", + "date": "\xa9day", + "year": "\xa9day", +} + + +def import_existing_track_for_album_slot(album_id: str, payload: dict, deps: MissingTrackImportDeps) -> dict: + source_track_id = payload.get("source_track_id") or payload.get("linked_track_id") + expected = payload.get("expected_track") or {} + if not source_track_id: + raise MissingTrackImportError("source_track_id is required", 400) + if not expected.get("track_number") or not (expected.get("title") or expected.get("name")): + raise MissingTrackImportError("expected_track with title and track_number is required", 400) + + database = deps.database + album_data, source_track = _load_album_and_source_track(database, album_id, source_track_id) + if album_data.get("server_source") and source_track.get("server_source") and album_data["server_source"] != source_track["server_source"]: + raise MissingTrackImportError("Selected track belongs to a different library source", 400) + + source_path = deps.resolve_library_file_path_fn(source_track.get("file_path")) + if not source_path: + raise MissingTrackImportError(_file_not_found_message(source_track.get("file_path")), 404) + + staging_path = _copy_source_to_staging(source_path, album_id, expected, deps) + metadata_source = (expected.get("source") or payload.get("source") or "").strip().lower() or "library" + expected_title = expected.get("title") or expected.get("name") or "Unknown Track" + expected_track_id = _expected_track_id(expected) + album_source_id = _album_source_id(payload, expected, album_data, album_id) + + api_track = _build_api_track(expected, expected_title, expected_track_id, album_source_id, metadata_source, album_data) + context = _build_context(payload, album_data, source_path, source_track_id, api_track, album_source_id, metadata_source) + + context_key = f"existing_import_{album_id}_{api_track['disc_number']}_{api_track['track_number']}_{uuid.uuid4().hex[:8]}" + deps.post_process_fn(context_key, context, staging_path) + final_path = context.get("_final_processed_path") + if not final_path or not os.path.exists(final_path): + raise MissingTrackImportError("Post-processing did not produce a final file", 500) + + ensure_tracks_disc_number_column(database) + copy_album_identity_from_target_sibling( + database, + album_id, + final_path, + api_track["disc_number"], + api_track["track_number"], + deps.resolve_library_file_path_fn, + ) + + target_track_id = _upsert_target_track( + database, + deps, + album_id, + album_data, + source_track, + final_path, + expected_title, + expected_track_id, + metadata_source, + api_track, + ) + _sync_imported_track(deps, target_track_id, expected_title, album_data) + + return { + "track_id": target_track_id, + "final_path": final_path, + "artist_id": album_data.get("target_artist_id"), + } + + +def _load_album_and_source_track(database, album_id: str, source_track_id: str) -> tuple[dict, dict]: + with database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT al.*, ar.name AS artist_name, ar.id AS target_artist_id + FROM albums al + JOIN artists ar ON ar.id = al.artist_id + WHERE al.id = ? + """, + (album_id,), + ) + album_row = cursor.fetchone() + if not album_row: + raise MissingTrackImportError("Album not found", 404) + + cursor.execute("SELECT * FROM tracks WHERE id = ?", (source_track_id,)) + source_row = cursor.fetchone() + if not source_row: + raise MissingTrackImportError("Selected library track not found", 404) + + return dict(album_row), dict(source_row) + + +def _copy_source_to_staging(source_path: str, album_id: str, expected: dict, deps: MissingTrackImportDeps) -> str: + download_dir = deps.docker_resolve_path_fn(deps.config_manager.get("soulseek.download_path", "./downloads")) + staging_root = os.path.join(download_dir, "ssync_existing_import") + os.makedirs(staging_root, exist_ok=True) + source_ext = os.path.splitext(source_path)[1] or ".audio" + staging_name = ( + f"existing_{album_id}_{expected.get('disc_number') or 1}_" + f"{expected.get('track_number')}_{uuid.uuid4().hex[:8]}{source_ext}" + ) + staging_path = os.path.join(staging_root, staging_name) + shutil.copy2(source_path, staging_path) + return staging_path + + +def _build_api_track( + expected: dict, + expected_title: str, + expected_track_id: str, + album_source_id: str, + metadata_source: str, + album_data: dict, +) -> dict: + return { + "id": expected_track_id, + "track_id": expected_track_id, + "name": expected_title, + "title": expected_title, + "track_number": int(expected.get("track_number") or 1), + "disc_number": int(expected.get("disc_number") or 1), + "duration_ms": int(expected.get("duration") or expected.get("duration_ms") or 0), + "artists": expected.get("artists") or [album_data.get("artist_name") or ""], + "source": metadata_source, + "album_id": album_source_id, + "spotify_track_id": expected.get("spotify_track_id") or "", + "deezer_id": expected.get("deezer_id") or "", + "itunes_track_id": expected.get("itunes_track_id") or "", + "musicbrainz_recording_id": expected.get("musicbrainz_recording_id") or "", + } + + +def _build_context( + payload: dict, + album_data: dict, + source_path: str, + source_track_id: str, + api_track: dict, + album_source_id: str, + metadata_source: str, +) -> dict: + api_album = { + "id": album_source_id, + "name": album_data.get("title") or "", + "title": album_data.get("title") or "", + "release_date": f"{album_data.get('year')}-01-01" if album_data.get("year") else "", + "total_tracks": album_data.get("api_track_count") or album_data.get("track_count") or 0, + "image_url": album_data.get("thumb_url") or "", + "source": metadata_source, + } + context = _build_post_process_context( + api_album, + api_track, + album_data.get("artist_name") or "", + album_data.get("title") or "", + int(payload.get("total_discs") or payload.get("expected_track", {}).get("total_discs") or 1), + ) + context["source"] = metadata_source + context["source_service"] = "existing_library" + context["source_filename"] = os.path.basename(source_path) + context["source_size"] = os.path.getsize(source_path) if os.path.exists(source_path) else 0 + context["explicit_album_context"] = True + context["from_existing_library_track"] = True + context["batch_id"] = f"existing_import_{album_data.get('id')}_{uuid.uuid4().hex[:8]}" + context["task_id"] = f"existing_import_{source_track_id}" + return context + + +def _upsert_target_track( + database, + deps: MissingTrackImportDeps, + album_id: str, + album_data: dict, + source_track: dict, + final_path: str, + expected_title: str, + expected_track_id: str, + metadata_source: str, + api_track: dict, +): + file_size, bitrate = _read_file_stats(final_path, source_track) + server_source = album_data.get("server_source") or source_track.get("server_source") or deps.config_manager.get_active_media_server() + + with database._get_connection() as conn: + cursor = conn.cursor() + _ensure_disc_number_column(cursor, conn) + + cursor.execute("SELECT id FROM tracks WHERE file_path = ? LIMIT 1", (final_path,)) + existing_by_path = cursor.fetchone() + cursor.execute( + """ + SELECT id FROM tracks + WHERE album_id = ? AND COALESCE(disc_number, 1) = ? AND track_number = ? + LIMIT 1 + """, + (album_id, api_track["disc_number"], api_track["track_number"]), + ) + existing_target = cursor.fetchone() + + if existing_by_path: + target_track_id = existing_by_path["id"] + cursor.execute( + """ + UPDATE tracks + SET album_id = ?, artist_id = ?, title = ?, track_number = ?, disc_number = ?, + duration = ?, file_path = ?, bitrate = ?, file_size = ?, + server_source = COALESCE(server_source, ?), + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + ( + album_id, + album_data.get("target_artist_id"), + expected_title, + api_track["track_number"], + api_track["disc_number"], + api_track["duration_ms"], + final_path, + bitrate, + file_size, + server_source, + target_track_id, + ), + ) + elif existing_target: + target_track_id = existing_target["id"] + cursor.execute( + """ + UPDATE tracks + SET title = ?, duration = ?, file_path = ?, bitrate = ?, file_size = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (expected_title, api_track["duration_ms"], final_path, bitrate, file_size, target_track_id), + ) + else: + cursor.execute("SELECT COALESCE(MAX(CAST(id AS INTEGER)), 0) + 1 AS next_id FROM tracks") + target_track_id = cursor.fetchone()["next_id"] + cursor.execute( + """ + INSERT INTO tracks ( + id, album_id, artist_id, title, track_number, disc_number, duration, + file_path, bitrate, file_size, server_source, created_at, updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + ( + target_track_id, + album_id, + album_data.get("target_artist_id"), + expected_title, + api_track["track_number"], + api_track["disc_number"], + api_track["duration_ms"], + final_path, + bitrate, + file_size, + server_source, + ), + ) + + track_source_col = (deps.service_id_columns or {}).get(metadata_source, {}).get("track") + if track_source_col and expected_track_id: + try: + cursor.execute(f"UPDATE tracks SET {track_source_col} = ? WHERE id = ?", (expected_track_id, target_track_id)) + except Exception as source_err: + logger.debug("Imported track source-id update failed: %s", source_err) + + conn.commit() + + return target_track_id + + +def _ensure_disc_number_column(cursor, conn) -> None: + cursor.execute("PRAGMA table_info(tracks)") + track_columns = {row[1] for row in cursor.fetchall()} + if "disc_number" not in track_columns: + cursor.execute("ALTER TABLE tracks ADD COLUMN disc_number INTEGER DEFAULT 1") + conn.commit() + + +def ensure_tracks_disc_number_column(database) -> None: + with database._get_connection() as conn: + cursor = conn.cursor() + _ensure_disc_number_column(cursor, conn) + + +def _read_file_stats(final_path: str, source_track: dict) -> tuple[Optional[int], int]: + file_size = None + bitrate = source_track.get("bitrate") or 0 + try: + file_size = os.path.getsize(final_path) + from mutagen import File as MutagenFile + + audio = MutagenFile(final_path) + if audio and getattr(audio, "info", None) and getattr(audio.info, "bitrate", None): + bitrate = int(audio.info.bitrate / 1000) + except Exception as meta_err: + logger.debug("Existing-track import metadata read failed: %s", meta_err) + return file_size, bitrate + + +def _sync_imported_track(deps: MissingTrackImportDeps, track_id, expected_title: str, album_data: dict) -> None: + try: + active_server = deps.config_manager.get_active_media_server() + if deps.sync_tracks_to_server_fn and active_server in ("jellyfin", "navidrome"): + deps.sync_tracks_to_server_fn( + [ + { + "id": track_id, + "title": expected_title, + "artist_name": album_data.get("artist_name"), + "album_title": album_data.get("title"), + "year": album_data.get("year"), + "server_source": album_data.get("server_source"), + } + ], + active_server, + ) + except Exception as sync_err: + logger.debug("Existing-track import server sync skipped/failed: %s", sync_err) + + +def copy_album_identity_from_target_sibling( + database, + album_id: str, + final_path: str, + target_disc: int, + target_track: int, + resolve_library_file_path_fn: Callable[[Optional[str]], Optional[str]], +) -> bool: + try: + with database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT file_path FROM tracks + WHERE album_id = ? + AND file_path IS NOT NULL + AND file_path != '' + AND NOT (COALESCE(disc_number, 1) = ? AND track_number = ?) + ORDER BY COALESCE(disc_number, 1), track_number + LIMIT 12 + """, + (album_id, target_disc, target_track), + ) + sibling_rows = cursor.fetchall() + + for row in sibling_rows: + sibling_path = resolve_library_file_path_fn(row["file_path"]) + if not sibling_path or not os.path.exists(sibling_path): + continue + tags = read_album_identity_tags(sibling_path) + if not tags: + continue + if write_album_identity_tags(final_path, tags): + logger.info("Imported track inherited album identity tags from sibling: %s", os.path.basename(sibling_path)) + return True + except Exception as exc: + logger.warning("Failed to inherit album identity tags for imported track: %s", exc) + return False + + +def read_album_identity_tags(file_path: str) -> dict: + try: + from mutagen import File as MutagenFile + from mutagen.id3 import ID3, TXXX + from mutagen.mp4 import MP4 + + audio = MutagenFile(file_path) + if not audio: + return {} + + tags = {} + if isinstance(getattr(audio, "tags", None), ID3): + for tag_key, frame_id in _ID3_STANDARD_TAGS.items(): + frames = audio.tags.getall(frame_id) + if frames and getattr(frames[0], "text", None): + tags[tag_key] = str(frames[0].text[0]).strip() + desc_to_key = {desc: key for key, desc in _ID3_TXXX_DESCS.items()} + for frame in audio.tags.getall("TXXX"): + if isinstance(frame, TXXX) and frame.desc in desc_to_key and frame.text: + tags[desc_to_key[frame.desc]] = str(frame.text[0]).strip() + elif isinstance(audio, MP4): + for tag_key, mp4_key in _MP4_STANDARD_TAGS.items(): + value = _first_tag_value(audio, mp4_key) + if value: + tags[tag_key] = value + for tag_key, desc in _ID3_TXXX_DESCS.items(): + value = _first_tag_value(audio, f"----:com.apple.iTunes:{desc}") + if value: + tags[tag_key] = value + else: + for tag_key in _ALBUM_IDENTITY_TAGS: + value = _first_tag_value(audio, tag_key) + if value: + tags[tag_key] = value + return {k: v for k, v in tags.items() if v} + except Exception as exc: + logger.debug("Failed reading album identity tags from %s: %s", file_path, exc) + return {} + + +def write_album_identity_tags(file_path: str, tags: dict) -> bool: + if not tags: + return False + try: + from mutagen import File as MutagenFile + from mutagen.id3 import ID3, TALB, TDRC, TPE2, TXXX + from mutagen.mp4 import MP4, MP4FreeForm + + audio = MutagenFile(file_path) + if not audio: + return False + + tags = {k: str(v).strip() for k, v in tags.items() if k in _ALBUM_IDENTITY_TAGS and str(v).strip()} + if not tags: + return False + + if isinstance(getattr(audio, "tags", None), ID3): + standard_frames = {"TALB": TALB, "TPE2": TPE2, "TDRC": TDRC} + written_standard = set() + for tag_key, frame_id in _ID3_STANDARD_TAGS.items(): + value = tags.get(tag_key) + if not value or frame_id in written_standard: + continue + audio.tags.delall(frame_id) + audio.tags.add(standard_frames[frame_id](encoding=3, text=[value])) + written_standard.add(frame_id) + for tag_key, desc in _ID3_TXXX_DESCS.items(): + value = tags.get(tag_key) + if not value: + continue + for existing in list(audio.tags.getall("TXXX")): + if getattr(existing, "desc", None) == desc: + audio.tags.remove(existing.HashKey) + audio.tags.add(TXXX(encoding=3, desc=desc, text=[value])) + elif isinstance(audio, MP4): + for tag_key, mp4_key in _MP4_STANDARD_TAGS.items(): + if tags.get(tag_key): + audio[mp4_key] = [tags[tag_key]] + for tag_key, desc in _ID3_TXXX_DESCS.items(): + if tags.get(tag_key): + audio[f"----:com.apple.iTunes:{desc}"] = [MP4FreeForm(tags[tag_key].encode("utf-8"))] + else: + for tag_key, value in tags.items(): + audio[tag_key] = [value] + + audio.save() + return True + except Exception as exc: + logger.warning("Failed writing album identity tags to %s: %s", file_path, exc) + return False + + +def _first_tag_value(audio, key: str) -> Optional[str]: + try: + values = audio.get(key) + if not values: + return None + value = values[0] if isinstance(values, (list, tuple)) else values + if isinstance(value, bytes): + value = value.decode("utf-8", errors="ignore") + return str(value).strip() or None + except Exception: + return None + + +def _expected_track_id(expected: dict) -> str: + return ( + expected.get("track_id") + or expected.get("id") + or expected.get("source_track_id") + or expected.get("spotify_track_id") + or expected.get("deezer_id") + or expected.get("itunes_track_id") + or expected.get("musicbrainz_recording_id") + or "" + ) + + +def _album_source_id(payload: dict, expected: dict, album_data: dict, album_id: str) -> str: + return ( + payload.get("album_source_id") + or expected.get("album_id") + or album_data.get("spotify_album_id") + or album_data.get("deezer_id") + or album_data.get("itunes_album_id") + or album_data.get("musicbrainz_release_id") + or album_data.get("discogs_id") + or album_data.get("tidal_id") + or album_data.get("qobuz_id") + or str(album_id) + ) + + +def _file_not_found_message(file_path: Optional[str]) -> str: + if file_path: + return f"File not found: {file_path}" + return "Selected library track does not have a file path" diff --git a/core/library/reorganize_tag_source.py b/core/library/reorganize_tag_source.py new file mode 100644 index 00000000..de83e9f2 --- /dev/null +++ b/core/library/reorganize_tag_source.py @@ -0,0 +1,327 @@ +"""Build reorganize-planning metadata from a file's embedded tags +instead of from a live metadata-source API call. + +Issue #592 (tacobell444): when a library has been carefully enriched ++ tagged, doing a fresh API lookup at reorganize time can introduce +inconsistencies (provider naming drift, version-mismatches, missing +album-level metadata for niche releases). The user's own embedded +tags are usually the most stable source of truth for an enriched +library — and using them costs zero API calls. + +This module is the pure tag-to-context adapter. It turns the dict +that ``core.library.file_tags.read_embedded_tags`` returns into the +``api_album`` / ``api_track`` shapes that +``library_reorganize._build_post_process_context`` already consumes. +That keeps the downstream pipeline path-builder, post-process +helpers, AcoustID, etc.) completely unchanged: tag-mode just produces +the same input shape via a different upstream route. + +Pure helpers — no IO inside the extractors so every shape is +test-pinnable. The wrapper :func:`read_album_track_from_file` does +the file IO via ``read_embedded_tags`` and then routes through the +extractors. + +Returns ``None`` (extractors) / ``(None, None, reason)`` (wrapper) +when the embedded tags are missing fields essential for reorganize +(track title, album name, or track artist). The plan layer surfaces +that as an unmatched item with a clear reason — same UX as when the +metadata-API call returns no candidate. No silent degradation.""" + +from __future__ import annotations + +import os +import re +from typing import Any, Dict, List, Optional, Tuple + + +# Tokens we accept as valid `releasetype` / `albumtype` values. +# Mirrors the canonical set the rest of the metadata pipeline uses +# (`core/metadata/album_tracks.py:_normalize_album_type`). +_VALID_ALBUM_TYPES = frozenset({'album', 'single', 'ep', 'compilation'}) + + +# Match a 4-digit year anywhere in a date-like string ("2020", +# "2020-01-15", "2020/01/15", "Jan 5, 2020", etc.). +_YEAR_RE = re.compile(r'(\d{4})') + + +# Separators we split a single artist field on to recover a list. +# Mirrors the same separator set ``core/metadata/artist_resolution.py`` +# uses when normalizing soulseek matched-download artist strings. +_ARTIST_SPLIT_RE = re.compile( + r'\s*(?:,|;|/|&| feat\. | feat | ft\. | ft | featuring | x | with )\s*', + re.IGNORECASE, +) + + +def _stringify(value: Any) -> str: + """Coerce an embedded-tag value into a clean string.""" + if value is None: + return '' + return str(value).strip() + + +def _parse_int_first(value: Any) -> Optional[int]: + """Parse a track/disc number that may arrive as ``"5"``, ``"5/12"``, + ``5``, ``5.0`` or even ``"05"``. Returns the leading integer, or + ``None`` when no integer is recoverable. + + Defensive against the trailing-``/N`` shape ID3 stores: ``TRCK = + "5/12"`` means "track 5 of 12", and we want ``5``.""" + if value is None: + return None + if isinstance(value, (int,)): + return value + if isinstance(value, float): + return int(value) + s = _stringify(value) + if not s: + return None + head = s.split('/', 1)[0].strip() + try: + return int(head) + except (TypeError, ValueError): + try: + return int(float(head)) + except (TypeError, ValueError): + return None + + +def _parse_int_total(value: Any) -> Optional[int]: + """Parse the trailing ``N`` of an ID3-style ``"5/12"`` value, or + return the parsed value when it's a plain integer string.""" + if value is None: + return None + if isinstance(value, int): + return value + s = _stringify(value) + if not s: + return None + if '/' in s: + tail = s.split('/', 1)[1].strip() + try: + return int(tail) + except (TypeError, ValueError): + return None + try: + return int(s) + except (TypeError, ValueError): + return None + + +def _normalize_year(value: Any) -> str: + """Extract a 4-digit year from a date-like field. Returns '' when + no year is extractable. Reorganize templates only use the year + portion of release dates, so we don't need to preserve the full + date string.""" + s = _stringify(value) + if not s: + return '' + m = _YEAR_RE.search(s) + return m.group(1) if m else '' + + +def _normalize_album_type(value: Any) -> str: + """Lowercase + validate the ``releasetype`` tag against the canonical + token set. Returns '' for unknown values so the downstream path + builder falls back to its default.""" + s = _stringify(value).lower() + if s in _VALID_ALBUM_TYPES: + return s + return '' + + +def _split_artists(value: Any) -> List[str]: + """Split an artist-string field into a list. Handles common + separators (``,``, ``;``, ``/``, ``&``, ``feat``, ``ft``, ``x``, + ``with``). Strips whitespace, drops empties, dedupes (case- + insensitive) while preserving order.""" + s = _stringify(value) + if not s: + return [] + parts = _ARTIST_SPLIT_RE.split(s) + seen: set = set() + out: List[str] = [] + for p in parts: + cleaned = p.strip() + if not cleaned: + continue + key = cleaned.lower() + if key in seen: + continue + seen.add(key) + out.append(cleaned) + return out + + +def _resolve_track_artists(tags: Dict[str, Any]) -> List[str]: + """Resolve the per-track artist list from embedded tags. Prefers a + multi-value ``artists`` tag (TXXX:Artists / Vorbis ``artists``) + over splitting the single-string ``artist`` tag, which is exactly + the precedence the post-download enrichment uses.""" + artists_value = tags.get('artists') + if artists_value: + # Multi-value tag readers may already have joined with ', '. + # Re-split to recover the list. + parts = _split_artists(artists_value) + if parts: + return parts + return _split_artists(tags.get('artist') or '') + + +def extract_track_meta_from_tags(tags: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Build an ``api_track``-shaped dict from embedded tags. + + Returns ``None`` if essential fields are missing (title or + artist). Caller surfaces that as an unmatched plan item. + + Output shape matches what ``library_reorganize._build_post_process_context`` + consumes (``name`` / ``track_number`` / ``disc_number`` / + ``artists`` / ``duration_ms`` / ``id``).""" + if not isinstance(tags, dict) or not tags: + return None + + title = _stringify(tags.get('title')) + if not title: + return None + + artists = _resolve_track_artists(tags) + if not artists: + return None + + track_number = _parse_int_first(tags.get('tracknumber')) or 1 + disc_number = _parse_int_first(tags.get('discnumber')) or 1 + + return { + 'name': title, + 'title': title, # belt-and-braces — both keys are read downstream + 'track_number': track_number, + 'disc_number': disc_number, + 'artists': [{'name': a} for a in artists], + 'duration_ms': 0, # not derivable from tags alone; set later from `duration` + 'id': '', # tag-mode has no source ID; reorganize doesn't need one + 'uri': '', + } + + +def extract_album_meta_from_tags(tags: Dict[str, Any]) -> Dict[str, Any]: + """Build an ``api_album``-shaped dict from embedded tags. + + Falls back to empty / zero values when fields are missing — the + path builder accepts those and uses its own defaults. The album + name is the only field we can't fall back on; if missing the + caller should treat the track as unmatched (handled by + :func:`read_album_track_from_file`).""" + if not isinstance(tags, dict): + tags = {} + + album_name = _stringify(tags.get('album')) + album_artist = _stringify(tags.get('albumartist') or tags.get('album_artist')) + release_date = _normalize_year(tags.get('date') or tags.get('year') or tags.get('originaldate')) + total_tracks = ( + _parse_int_total(tags.get('totaltracks')) + or _parse_int_total(tags.get('tracktotal')) + or _parse_int_total(tags.get('tracknumber')) # may be "5/12" + or 0 + ) + album_type = _normalize_album_type(tags.get('releasetype')) + + # `total_discs` only comes from explicit total signals: a + # `totaldiscs` tag, or the trailing `/N` of an ID3-style + # `discnumber = "1/2"`. A bare `discnumber = "1"` carries no total + # and must NOT be treated as one (else single-disc albums would + # claim total=1 and the path builder would still skip the + # subfolder, but partial-album cases would underreport). + total_discs = _parse_int_total(tags.get('totaldiscs')) or 0 + discnumber_raw = _stringify(tags.get('discnumber')) + if '/' in discnumber_raw: + explicit_total = _parse_int_total(discnumber_raw) + if explicit_total: + total_discs = max(total_discs, explicit_total) + + return { + 'id': '', + 'album_id': '', + 'name': album_name, + 'title': album_name, + 'release_date': release_date, + 'total_tracks': total_tracks, + 'total_discs': total_discs, + 'image_url': '', + 'images': [], + 'album_artist': album_artist, + 'album_type': album_type, + } + + +def read_album_track_from_file( + file_path: str, + *, + read_embedded_tags_fn=None, +) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]], Optional[str]]: + """Read embedded tags from ``file_path`` and produce + ``(album_meta, track_meta, error_reason)``. + + Returns ``(None, None, reason)`` when the file can't be opened, + has no recognisable tags, or is missing essential fields (title + or artist). The reason string is human-readable and suitable for + surfacing directly in the reorganize preview/error UI. + + Args: + file_path: Resolved on-disk path to the audio file. + read_embedded_tags_fn: Optional override for the tag reader, + used by tests to avoid real mutagen IO. Defaults to + ``core.library.file_tags.read_embedded_tags``.""" + if not file_path or not isinstance(file_path, str): + return None, None, 'No file path on track row.' + + if read_embedded_tags_fn is None: + from core.library.file_tags import read_embedded_tags as _real_reader + read_embedded_tags_fn = _real_reader + + result = read_embedded_tags_fn(file_path) + if not isinstance(result, dict) or not result.get('available'): + reason = (result or {}).get('reason') if isinstance(result, dict) else None + return None, None, reason or 'Could not read embedded tags from file.' + + tags = result.get('tags') or {} + track_meta = extract_track_meta_from_tags(tags) + if track_meta is None: + return None, None, 'Embedded tags missing required title or artist.' + + album_meta = extract_album_meta_from_tags(tags) + if not album_meta.get('name'): + return None, None, 'Embedded tags missing album name.' + + # Promote duration from the file-info block onto the track meta + # so the path builder has a non-zero value if a downstream + # consumer wants it. + duration_seconds = result.get('duration') or 0 + try: + track_meta['duration_ms'] = int(float(duration_seconds) * 1000) + except (TypeError, ValueError): + track_meta['duration_ms'] = 0 + + return album_meta, track_meta, None + + +def normalize_resolved_path(file_path: Optional[str]) -> Optional[str]: + """Defensive wrapper: returns the input only when it points at a + real file. Saves the caller from another ``os.path.exists`` check + in already-noisy code paths.""" + if not file_path: + return None + try: + if not os.path.exists(file_path): + return None + except OSError: + return None + return file_path + + +__all__ = [ + 'extract_track_meta_from_tags', + 'extract_album_meta_from_tags', + 'read_album_track_from_file', + 'normalize_resolved_path', +] diff --git a/core/library/retag.py b/core/library/retag.py index 48be9800..f5cf74e7 100644 --- a/core/library/retag.py +++ b/core/library/retag.py @@ -37,7 +37,7 @@ import os import traceback from dataclasses import dataclass from difflib import SequenceMatcher -from typing import Any, Callable +from typing import Any, Callable, Optional logger = logging.getLogger(__name__) @@ -62,6 +62,13 @@ class RetagDeps: _get_retag_state: Callable[[], dict] _set_retag_state: Callable[[dict], None] get_database: Callable[[], Any] + # Discord report (Netti93) — retag was clearing the LYRICS / USLT + # tag without rewriting it, while the download pipeline calls + # `generate_lrc_file` after enrichment to refetch + embed lyrics. + # Injected here so retag mirrors the same post-enrichment step. + # Optional for backward compat with any test caller that builds + # RetagDeps without the new field — empty default no-ops the call. + generate_lrc_file: Optional[Callable] = None @property def retag_state(self) -> dict: @@ -230,6 +237,20 @@ def execute_retag(group_id, album_id, deps: RetagDeps): except Exception as meta_err: logger.error(f"[Retag] Metadata write failed for '{track_title}': {meta_err}") + # Discord report (Netti93) — `enhance_file_metadata` clears + # ALL tags (incl. USLT lyrics) and rewrites only the source + # metadata. The download pipeline calls `generate_lrc_file` + # after enrichment to refetch + embed lyrics — retag was + # missing that step and dropped the LYRICS tag with no + # rewrite. Mirroring the download path's post-enrichment + # step. Same args, same `lrclib_enabled` config gate, same + # idempotency (skip when sidecar already present). + if deps.generate_lrc_file: + try: + deps.generate_lrc_file(current_file_path, context, new_artist, album_info) + except Exception as lrc_err: + logger.debug("[Retag] generate_lrc_file failed for '%s': %s", track_title, lrc_err) + # Compute new path and move if different file_ext = os.path.splitext(current_file_path)[1] try: diff --git a/core/library/service_search.py b/core/library/service_search.py index fb47d48a..38fd450e 100644 --- a/core/library/service_search.py +++ b/core/library/service_search.py @@ -19,6 +19,7 @@ tidal_enrichment_worker = None qobuz_enrichment_worker = None discogs_worker = None audiodb_worker = None +amazon_worker = None def init( @@ -31,11 +32,12 @@ def init( qobuz_worker=None, discogs_worker_obj=None, audiodb_worker_obj=None, + amazon_worker_obj=None, ): """Bind enrichment worker handles so the lifted bodies can use them.""" global spotify_enrichment_worker, itunes_enrichment_worker, mb_worker global lastfm_worker, genius_worker, tidal_enrichment_worker - global qobuz_enrichment_worker, discogs_worker, audiodb_worker + global qobuz_enrichment_worker, discogs_worker, audiodb_worker, amazon_worker spotify_enrichment_worker = spotify_worker itunes_enrichment_worker = itunes_worker mb_worker = musicbrainz_worker @@ -45,6 +47,7 @@ def init( qobuz_enrichment_worker = qobuz_worker discogs_worker = discogs_worker_obj audiodb_worker = audiodb_worker_obj + amazon_worker = amazon_worker_obj def _detect_provider(items, client): @@ -97,12 +100,14 @@ def _search_service(service, entity_type, query): if not mb_worker or not mb_worker.mb_service: raise ValueError("MusicBrainz worker not initialized") mb_client = mb_worker.mb_service.mb_client + # User-facing manual search — prefer recall (fuzzy / alias / diacritic- + # folded) over strict phrase precision. User picks correct hit from list. if entity_type == 'artist': - items = mb_client.search_artist(query, limit=8) + items = mb_client.search_artist(query, limit=8, strict=False) return [{'id': a['id'], 'name': a.get('name', ''), 'image': None, 'extra': f"Score: {a.get('score', '')} · {a.get('disambiguation', '') or a.get('country', '')}"} for a in items] elif entity_type == 'album': - items = mb_client.search_release(query, limit=8) + items = mb_client.search_release(query, limit=8, strict=False) results = [] for r in items: artists = ', '.join(ac.get('name', '') for ac in r.get('artist-credit', []) if isinstance(ac, dict)) @@ -112,7 +117,7 @@ def _search_service(service, entity_type, query): 'extra': f"{artists} · {r.get('date', '')} · Score: {r.get('score', '')}"}) return results elif entity_type == 'track': - items = mb_client.search_recording(query, limit=8) + items = mb_client.search_recording(query, limit=8, strict=False) results = [] for r in items: artists = ', '.join(ac.get('name', '') for ac in r.get('artist-credit', []) if isinstance(ac, dict)) @@ -293,4 +298,22 @@ def _search_service(service, entity_type, query): 'image': None, 'extra': f"{result.get('strArtist', '')} · {result.get('strAlbum', '')}"}] return [] + elif service == 'amazon': + if not amazon_worker or not amazon_worker.client: + raise ValueError("Amazon worker not initialized") + client = amazon_worker.client + if entity_type == 'artist': + items = client.search_artists(query, limit=8) + return [{'id': str(a.id), 'name': a.name, 'image': a.image_url, + 'extra': ', '.join(a.genres[:3]) if a.genres else ''} for a in items] + elif entity_type == 'album': + items = client.search_albums(query, limit=8) + return [{'id': str(a.id), 'name': a.name, 'image': a.image_url, + 'extra': f"{', '.join(a.artists)} · {a.release_date or ''}"} for a in items] + elif entity_type == 'track': + items = client.search_tracks(query, limit=8) + return [{'id': str(t.id), 'name': t.name, 'image': t.image_url, + 'extra': f"{', '.join(t.artists)} · {t.album or ''}"} for t in items] + return [] + return [] diff --git a/core/library_reorganize.py b/core/library_reorganize.py index 36bbf763..2aea45bb 100644 --- a/core/library_reorganize.py +++ b/core/library_reorganize.py @@ -549,17 +549,139 @@ def load_album_and_tracks(db, album_id): pass +def _plan_from_tags( + album_data: dict, + tracks: List[dict], + resolve_file_path_fn: Optional[Callable[[Optional[str]], Optional[str]]], +) -> dict: + """Tag-mode planner: build per-track ``api_track`` shapes from each + file's own embedded metadata instead of a live source API call. + + Per-track behavior: + - File missing on disk → unmatched with reason. + - Tags missing essentials (title / artist / album) → unmatched + with reason. + - Otherwise matched with the per-file extracted ``api_track`` and + a per-file ``api_album``. The plan stores the FIRST matched + track's album dict on the top-level ``api_album`` field for + backward compatibility with downstream callers; downstream + consumers that need the per-track album shape read it off + ``items[i]['api_album']``. + + Returns the same status / source / api_album / total_discs / items + shape as :func:`plan_album_reorganize`. ``source`` is the literal + string ``'tags'`` so callers can distinguish from API sources.""" + if resolve_file_path_fn is None: + # Without the file-path resolver we can't read anything off + # disk. Return an unmatched plan so callers surface a clear + # error instead of silently returning empty. + reason = 'Tag-mode reorganize requires the file path resolver.' + return { + 'status': 'no_source_id', 'source': None, 'api_album': None, + 'total_discs': 1, + 'items': [{ + 'track': t, 'api_track': None, 'matched': False, + 'reason': reason, + } for t in tracks], + } + + from core.library.reorganize_tag_source import read_album_track_from_file + + items: List[dict] = [] + first_album_meta: Optional[dict] = None + max_disc = 1 + + for track in tracks: + db_path = track.get('file_path') + resolved = resolve_file_path_fn(db_path) if db_path else None + if not resolved: + items.append({ + 'track': track, 'api_track': None, 'api_album': None, + 'matched': False, + 'reason': 'File no longer exists on disk for this track.', + }) + continue + + album_meta, track_meta, err = read_album_track_from_file(resolved) + if err is not None or track_meta is None or album_meta is None: + items.append({ + 'track': track, 'api_track': None, 'api_album': None, + 'matched': False, + 'reason': err or 'Could not extract metadata from embedded tags.', + }) + continue + + if first_album_meta is None: + first_album_meta = album_meta + try: + disc = int(track_meta.get('disc_number') or 1) + except (TypeError, ValueError): + disc = 1 + if disc > max_disc: + max_disc = disc + # Respect an explicit `totaldiscs` tag (or "1/2" disc-number + # form) so a partial-album reorganize (only disc 1 present + # locally) still routes into `Disc 1/` when the file's tags + # know there are 2 discs total. + try: + tagged_total = int(album_meta.get('total_discs') or 0) + except (TypeError, ValueError): + tagged_total = 0 + if tagged_total > max_disc: + max_disc = tagged_total + + items.append({ + 'track': track, + 'api_track': track_meta, + 'api_album': album_meta, + 'matched': True, + 'reason': None, + }) + + if not any(it['matched'] for it in items): + return { + 'status': 'no_source_id', + 'source': 'tags', + 'api_album': None, + 'total_discs': 1, + 'items': items, + } + + return { + 'status': 'planned', + 'source': 'tags', + 'api_album': first_album_meta or {}, + 'total_discs': max_disc, + 'items': items, + } + + def plan_album_reorganize( album_data: dict, tracks: List[dict], primary_source: Optional[str] = None, strict_source: bool = False, + metadata_source: str = 'api', + resolve_file_path_fn: Optional[Callable[[Optional[str]], Optional[str]]] = None, ) -> dict: """Compute the per-track plan for an album reorganize without doing any file IO. Both the actual reorganize orchestrator and the preview endpoint share this so the preview is guaranteed to match what would happen on apply. + ``metadata_source``: + - ``'api'`` (default): query the configured metadata source(s) + for the canonical tracklist (existing behavior). Issues an + API call. + - ``'tags'``: read each file's embedded tags as the source of + truth (issue #592). Zero API calls; trusts the user's + enriched library. + + When ``metadata_source='tags'``, ``resolve_file_path_fn`` MUST be + provided (the planner needs to read the actual files). The + ``primary_source`` and ``strict_source`` params are ignored in + tag mode. + Returns: ``{'status': 'planned' | 'no_source_id' | 'no_tracks', 'source': str | None, @@ -581,6 +703,9 @@ def plan_album_reorganize( 'total_discs': 1, 'items': [], } + if metadata_source == 'tags': + return _plan_from_tags(album_data, tracks, resolve_file_path_fn) + if primary_source is None: try: primary_source = get_primary_source() @@ -720,6 +845,7 @@ def preview_album_reorganize( build_final_path_fn: Callable, primary_source: Optional[str] = None, strict_source: bool = False, + metadata_source: str = 'api', ) -> dict: """Compute the planned destination paths for a reorganize WITHOUT moving any files. The preview UI uses this to show users what the @@ -775,6 +901,8 @@ def preview_album_reorganize( plan = plan_album_reorganize( album_data, tracks, primary_source=primary_source, strict_source=strict_source, + metadata_source=metadata_source, + resolve_file_path_fn=resolve_file_path_fn, ) artist_name = album_data.get('artist_name') or 'Unknown Artist' album_title = album_data.get('title') or 'Unknown Album' @@ -836,8 +964,12 @@ def preview_album_reorganize( item['disc_number'] = int(api_track.get('disc_number') or 1) # Build the same context the orchestrator builds so the path # builder produces the same destination it would on apply. + # Tag-mode plan items carry per-item album metadata; fall back + # to the shared api_album in API mode (where every plan item + # shares the same one). + per_item_album = plan_item.get('api_album') or api_album context = _build_post_process_context( - api_album, api_track, artist_name, album_title, total_discs + per_item_album, api_track, artist_name, album_title, total_discs ) # `_build_final_path_for_track` switches between ALBUM and SINGLE # modes based on `album_info.get('is_album')` — must be passed, @@ -1034,13 +1166,18 @@ def _stage_track(ctx: _RunContext, track_id, title, resolved_src) -> Optional[st return staging_file -def _run_post_process_for_track(ctx: _RunContext, track_id, title, api_track, staging_file) -> Optional[str]: +def _run_post_process_for_track(ctx: _RunContext, track_id, title, api_track, staging_file, *, per_item_api_album=None) -> Optional[str]: """Build the per-track context, hand it to post-processing, and return the final on-disk path it produced. Returns None on any failure (exception, AcoustID rejection, internal skip); the caller - leaves the original file alone.""" + leaves the original file alone. + + ``per_item_api_album`` overrides ``ctx.api_album`` for this track — + used in tag-mode reorganize where each file may carry its own + embedded album metadata.""" + api_album = per_item_api_album if per_item_api_album else ctx.api_album context = _build_post_process_context( - ctx.api_album, api_track, ctx.artist_name, ctx.album_title, ctx.total_discs + api_album, api_track, ctx.artist_name, ctx.album_title, ctx.total_discs ) context_key = f"reorganize_{ctx.album_id}_{track_id}_{uuid.uuid4().hex[:8]}" try: @@ -1138,7 +1275,10 @@ def _process_one_track(ctx: _RunContext, plan_item: dict) -> None: if staging_file is None: return - new_path = _run_post_process_for_track(ctx, track_id, title, plan_item['api_track'], staging_file) + new_path = _run_post_process_for_track( + ctx, track_id, title, plan_item['api_track'], staging_file, + per_item_api_album=plan_item.get('api_album'), + ) if new_path is None: return @@ -1180,6 +1320,7 @@ def reorganize_album( primary_source: Optional[str] = None, strict_source: bool = False, stop_check: Optional[Callable[[], bool]] = None, + metadata_source: str = 'api', ) -> dict: """Run a single album through the post-processing pipeline. @@ -1257,10 +1398,19 @@ def reorganize_album( plan = plan_album_reorganize( album_data, tracks, primary_source=primary_source, strict_source=strict_source, + metadata_source=metadata_source, + resolve_file_path_fn=resolve_file_path_fn, ) if plan['status'] == 'no_source_id': summary['status'] = 'no_source_id' - if _is_unknown_artist(album_data.get('artist_name')) or _looks_like_album_id_title(album_data.get('title')): + summary['source'] = plan.get('source') # 'tags' or None + if plan.get('source') == 'tags': + err_text = ( + f"No tracks of '{album_data.get('title', '?')}' have readable " + "embedded tags (missing title / artist / album, or file unreadable). " + "Switch back to API mode or fix the embedded tags first." + ) + elif _is_unknown_artist(album_data.get('artist_name')) or _looks_like_album_id_title(album_data.get('title')): err_text = ( f"Album '{album_data.get('title', '?')}' has placeholder metadata " "(Unknown Artist or numeric title) — run the 'Fix Unknown Artists' " diff --git a/core/lyrics_client.py b/core/lyrics_client.py index e7efebd8..eb49f6fa 100644 --- a/core/lyrics_client.py +++ b/core/lyrics_client.py @@ -52,9 +52,29 @@ class LyricsClient: lrc_path = os.path.splitext(audio_file_path)[0] + '.lrc' txt_path = os.path.splitext(audio_file_path)[0] + '.txt' - # Skip if lyrics file already exists (either .lrc or .txt) + # Sidecar already exists — skip the LRClib fetch but still + # re-embed the lyrics in the audio file's tag. The retag + # flow clears all tags including USLT and then runs this + # helper to restore them; without the embed step the + # LYRICS tag stays empty even though the .lrc is right + # there next to the file (Discord report — Netti93). if os.path.exists(lrc_path) or os.path.exists(txt_path): - logger.debug(f"Lyrics file already exists for: {os.path.basename(audio_file_path)}") + existing_path = lrc_path if os.path.exists(lrc_path) else txt_path + try: + with open(existing_path, 'r', encoding='utf-8') as f: + existing_lyrics = f.read().strip() + if existing_lyrics: + self._embed_lyrics(audio_file_path, existing_lyrics) + logger.debug( + "Re-embedded lyrics from existing %s for: %s", + os.path.basename(existing_path), + os.path.basename(audio_file_path), + ) + except Exception as e: + logger.debug( + "Could not re-embed lyrics from existing sidecar %s: %s", + os.path.basename(existing_path), e, + ) return True # Fetch lyrics from LRClib diff --git a/core/matching/acoustid_candidates.py b/core/matching/acoustid_candidates.py new file mode 100644 index 00000000..eac2ee49 --- /dev/null +++ b/core/matching/acoustid_candidates.py @@ -0,0 +1,143 @@ +"""Find a matching AcoustID candidate for an expected (title, artist). + +AcoustID returns multiple recordings per fingerprint — same audio can +correspond to multiple MusicBrainz recordings (different releases, +different metadata-quality entries, sample / cover-version collisions). +The "top" recording AcoustID returns isn't always the one whose +metadata matches the user's expected track. + +Both the post-download verifier (`core/acoustid_verification.py`) and +the AcoustID library scanner (`core/repair_jobs/acoustid_scanner.py`) +need to ask: "given these candidates, does ANY of them match +(expected_title, expected_artist) by title+artist similarity?" The +verifier had its own inline loop; the scanner only checked the top +match → false positives whenever the wrong-credited recording out- +ranked the right-credited one. + +This module is the single shared boundary for that question. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, Iterable, Optional, Tuple + +from utils.logging_config import get_logger + +logger = get_logger("matching.acoustid_candidates") + + +def find_matching_recording( + recordings: Iterable[Dict[str, Any]], + expected_title: str, + expected_artist: str, + *, + title_threshold: float = 0.70, + artist_threshold: float = 0.60, + similarity: Optional[Callable[[str, str], float]] = None, + artist_similarity: Optional[Callable[[str, str], float]] = None, + skip_predicate: Optional[Callable[[Dict[str, Any]], bool]] = None, +) -> Tuple[Optional[Dict[str, Any]], float, float]: + """Return the first AcoustID candidate whose metadata passes both + title + artist similarity thresholds. + + Args: + recordings: AcoustID recording dicts. Each must carry ``title`` + and ``artist`` strings; entries without both are skipped. + expected_title: The track title the caller expected. + expected_artist: The artist the caller expected. + title_threshold: Minimum title similarity to accept (default 0.70). + artist_threshold: Minimum artist similarity to accept (default 0.60). + similarity: ``(a, b) -> float`` for title comparison. Defaults + to a lowercase exact-equals stub when not supplied — callers + should pass their stricter normaliser (verifier passes its + parenthetical-stripping ``_similarity``; scanner passes + its own). + artist_similarity: ``(expected, actual) -> float`` for artist + comparison. Lets callers supply alias-aware comparison + (verifier wraps ``_alias_aware_artist_sim``; scanner wraps + ``artist_names_match``). Defaults to ``similarity`` if + unset. + skip_predicate: Optional ``(recording_dict) -> bool``. When + truthy, the candidate is skipped (used by the verifier to + drop wrong-version recordings — instrumental vs vocal etc). + + Returns: + ``(recording, title_sim, artist_sim)`` for the first matching + candidate, or ``(None, best_title_sim, best_artist_sim)`` when + none match. The non-None ``best_*`` values let callers report + the closest near-miss when they need to log why nothing matched. + + Iteration order matches the input order (typically AcoustID's own + fingerprint-confidence ranking). Returns on first match — does NOT + score every candidate looking for the highest sim. + """ + if not expected_title or not expected_artist: + return None, 0.0, 0.0 + + sim = similarity or _default_similarity + asim = artist_similarity or sim + + best_title_sim = 0.0 + best_artist_sim = 0.0 + + for rec in recordings or (): + if not isinstance(rec, dict): + continue + rec_title = (rec.get('title') or '').strip() + rec_artist = (rec.get('artist') or '').strip() + if not rec_title or not rec_artist: + continue + if skip_predicate and skip_predicate(rec): + continue + + title_sim = sim(expected_title, rec_title) + if title_sim > best_title_sim: + best_title_sim = title_sim + + artist_sim = asim(expected_artist, rec_artist) + if artist_sim > best_artist_sim: + best_artist_sim = artist_sim + + if title_sim >= title_threshold and artist_sim >= artist_threshold: + return rec, title_sim, artist_sim + + return None, best_title_sim, best_artist_sim + + +def _default_similarity(a: str, b: str) -> float: + if not a or not b: + return 0.0 + return 1.0 if a.lower().strip() == b.lower().strip() else 0.0 + + +# ──────────────────────────────────────────────────────────────────── +# Duration guard — codex item (5). +# ──────────────────────────────────────────────────────────────────── + + +def duration_mismatches_strongly( + expected_seconds: Optional[float], + candidate_seconds: Optional[float], + *, + abs_tolerance_s: float = 60.0, + rel_tolerance: float = 0.35, +) -> bool: + """Return True when the candidate's duration is too far from expected + to confidently treat it as the same recording. + + Catches fingerprint hash collisions (the reporter's 17-minute + mashup → 5-minute Japanese hiphop track case). When EITHER duration + is unknown / non-positive, returns False — no behavior change. + + Threshold: drift greater than max(``abs_tolerance_s``, + ``rel_tolerance * expected``). The relative term scales with track + length so a 20% mismatch on a 3-minute track and a 20% mismatch on + a 30-minute mix are both treated as suspicious. + """ + if not expected_seconds or expected_seconds <= 0: + return False + if not candidate_seconds or candidate_seconds <= 0: + return False + drift = abs(float(candidate_seconds) - float(expected_seconds)) + threshold = max(abs_tolerance_s, rel_tolerance * float(expected_seconds)) + return drift > threshold diff --git a/core/matching/album_context_title.py b/core/matching/album_context_title.py new file mode 100644 index 00000000..7fd0d2cd --- /dev/null +++ b/core/matching/album_context_title.py @@ -0,0 +1,195 @@ +"""Strip redundant album-context suffixes from track titles. + +Issue #589 — MTV Unplugged albums (and similar live-concert / session +releases) have source-side track titles like ``"Shy Away (MTV Unplugged +Live)"`` while the local DB stored title is just ``"Shy Away"``. The +album-scoped library check at ``core/downloads/master.py`` compares +the two with raw string similarity, the length asymmetry tanks the +score, and tracks the user already owns get marked missing. + +This helper normalizes a track title by stripping the parenthetical +or dash suffix when its tokens are fully subsumed by the album +context: at least one version marker (live / unplugged / acoustic / +session / etc) is present in BOTH the suffix AND the album title, and +every other suffix token is either a known marker, a year, a +connecting noise word, or a word that appears in the album title. + +Pure function. No I/O. Tests at the function boundary. +""" + +from __future__ import annotations + +import re +from typing import Iterable, Tuple + +# Version-marker keywords. When the album title contains any of these, +# stripping is enabled. Singular forms — plurals get matched separately +# via stem expansion below. +_VERSION_MARKERS = ( + 'live', + 'unplugged', + 'acoustic', + 'session', + 'concert', + 'tour', +) + +# Markers that are implied "live" context — when the album mentions any +# of these, a bare ``live`` token in the suffix counts as album context +# even if the album title doesn't literally say "live". MTV Unplugged +# albums are live recordings; same for "in concert" / "tour" releases. +_IMPLIES_LIVE = ('unplugged', 'concert', 'tour', 'session') + +# Connecting / filler words that don't carry meaning by themselves. +_NOISE_TOKENS = frozenset({ + 'version', 'edition', 'recording', 'recordings', 'remaster', + 'remastered', 'mix', + 'the', 'a', 'an', 'from', 'at', 'in', 'on', 'for', 'of', + 'and', 'or', 'with', 'by', + 'vol', 'pt', 'part', 'no', +}) + +_SUFFIX_PATTERNS: Tuple[re.Pattern, ...] = ( + re.compile(r'\s*\(([^()]+)\)\s*$'), + re.compile(r'\s*\[([^\[\]]+)\]\s*$'), + re.compile(r'\s+-\s+(.+?)\s*$'), +) + +_YEAR_RE = re.compile(r'^(?:19|20)\d{2}$') +_TOKEN_RE = re.compile(r'\w+') + + +def _normalize(text: str) -> str: + return (text or '').lower().strip() + + +def _tokenize(text: str) -> set: + return set(_TOKEN_RE.findall(_normalize(text))) + + +def _expand_marker_set(markers: Iterable[str]) -> set: + """Expand each marker into its singular + plural forms.""" + out = set() + for marker in markers: + out.add(marker) + if not marker.endswith('s'): + out.add(marker + 's') + return out + + +_EXPANDED_MARKERS = _expand_marker_set(_VERSION_MARKERS) + + +def album_context_markers(album_title: str) -> Tuple[str, ...]: + """Return the version markers present in the album title (singular form).""" + if not album_title: + return () + album_tokens = _tokenize(album_title) + found = [] + for marker in _VERSION_MARKERS: + if marker in album_tokens or (marker + 's') in album_tokens: + found.append(marker) + return tuple(found) + + +def _suffix_is_album_redundant( + inner: str, + album_tokens: set, + album_markers: Tuple[str, ...], +) -> bool: + """Decide whether a suffix's tokens are all subsumed by album context. + + Three requirements: + 1. The suffix contains at least one version-marker token. Stops + a generic "feat. X" suffix from being stripped because the + album happened to be live. + 2. The shared marker matches one the album implies — either + literally in the album title, OR via the implied-live set + (unplugged/concert/tour albums imply "live"). + 3. Every other suffix token is either a marker, a year, a + tolerated noise word, or a word that appears in the album + title. If any token falls outside, the suffix carries + info beyond album context (featured artist, different + version, etc) — keep it on. + """ + if not inner: + return False + + suffix_tokens = _tokenize(inner) + if not suffix_tokens: + return False + + # Markers the album effectively implies (literal + implied-live). + implied_markers = set(album_markers) + if any(m in implied_markers for m in _IMPLIES_LIVE): + implied_markers.add('live') + + suffix_markers = suffix_tokens & _EXPANDED_MARKERS + if not suffix_markers: + return False + + # At least one marker must overlap with album-implied set. Plural + # tolerance — strip trailing 's' for the comparison. + def _stem(tok: str) -> str: + return tok[:-1] if tok.endswith('s') and len(tok) > 1 else tok + + if not any(_stem(t) in implied_markers for t in suffix_markers): + return False + + # Every remaining suffix token must be subsumed. + for tok in suffix_tokens: + if tok in _EXPANDED_MARKERS: + continue + if _YEAR_RE.match(tok): + continue + if tok in _NOISE_TOKENS: + continue + if tok in album_tokens: + continue + return False + + return True + + +def strip_redundant_album_suffix(track_title: str, album_title: str) -> str: + """Strip a trailing parenthetical/bracket/dash suffix from `track_title` + when the suffix duplicates context already implied by `album_title`. + + Examples: + - ("Shy Away (MTV Unplugged Live)", "MTV Unplugged") → "Shy Away" + - ("Only If For A Night (MTV Unplugged, 2012 / Live)", + "Ceremonials (Live At MTV Unplugged)") → "Only If For A Night" + - ("In My Feelings (Instrumental)", "Scorpion") + → unchanged (instrumental NOT implied by studio album) + - ("Hello (Live - feat. Other)", "Live At Wembley") + → unchanged (suffix carries featured-artist beyond album context) + - ("Shy Away", "MTV Unplugged") → unchanged (no suffix) + + Pure function — never raises, returns the input unchanged on any + edge / unexpected input. + """ + if not track_title: + return track_title or '' + album_markers = album_context_markers(album_title) + if not album_markers: + return track_title + + album_tokens = _tokenize(album_title) + stripped = track_title + + # Stacked suffixes ("Track (MTV Unplugged) [Live]") — peel one at a + # time. Bound the loop defensively. + for _ in range(4): + peeled = None + for pattern in _SUFFIX_PATTERNS: + m = pattern.search(stripped) + if not m: + continue + inner = m.group(1) + if _suffix_is_album_redundant(inner, album_tokens, album_markers): + peeled = stripped[: m.start()].rstrip() + break + if peeled is None: + return stripped + stripped = peeled + return stripped diff --git a/core/matching/version_mismatch.py b/core/matching/version_mismatch.py new file mode 100644 index 00000000..0b60028c --- /dev/null +++ b/core/matching/version_mismatch.py @@ -0,0 +1,115 @@ +"""Decide when an AcoustID version-annotation mismatch should still +pass verification. + +Issue #607 (AfonsoG6): live recordings were quarantining as +"Version mismatch: expected '... (Live at Venue)' (live) but file is +'Song' (original)" because MusicBrainz often stores the recording +entity with a bare title — the venue / live annotation lives on the +release entity, not the recording. The audio fingerprint correctly +identifies the live recording (live audio has its own distinct +fingerprint), but the title-text comparison flagged it as the wrong +version. + +Strict version mismatching stays for genuinely-different recordings +(instrumental vs vocal, remix vs original, acoustic vs studio) — +those have distinct fingerprints AND MB always annotates them in the +recording title. We only loosen for the **live** direction +specifically, since that's the asymmetry users actually hit: + +- LIVE: MB often stores live recordings with bare titles. The + fingerprint correctly identifies the live recording even when the + recording's text title lacks a "(Live at ...)" annotation. +- INSTRUMENTAL / REMIX / ACOUSTIC / DEMO / etc: MB always carries + the version marker in the recording title. If AcoustID returns an + instrumental for a vocal file query (or vice versa), it's a real + wrong-recording match — the fingerprint matched the instrumental + audio, which IS the wrong file. + +Two-sided version mismatches stay strict — "live" vs "remix" really +are different recordings even if MB titled them similarly.""" + +from __future__ import annotations + + +_BARE_VERSION = 'original' + +# Versions where a one-sided annotation difference is plausibly +# explained by MB metadata gaps rather than a different recording. +# Live recordings are the primary case; venue annotations live on the +# release-track entity, not the recording entity, so the recording can +# be bare-titled even though the audio is genuinely live. +_LIVE_AWARE_VERSIONS = frozenset({'live'}) + + +def is_acceptable_version_mismatch( + expected_version: str, + matched_version: str, + *, + fingerprint_score: float, + title_similarity: float, + artist_similarity: float, + score_threshold: float = 0.85, + title_threshold: float = 0.70, + artist_threshold: float = 0.60, +) -> bool: + """Return True when an expected-vs-matched version annotation + difference is likely a MusicBrainz metadata gap rather than a + genuinely different recording. + + Conditions (all must hold): + + 1. The mismatch is **one-sided AND involves a live-aware version** + — exactly one side is ``'live'`` and the other is bare + ``'original'``. Other version markers (instrumental, remix, + acoustic, etc) carry distinct fingerprints AND are always + annotated in MB's recording title — we don't loosen for them. + 2. Fingerprint score ``>= score_threshold`` (default 0.85). The + AcoustID fingerprint is high-confidence — we trust it. + 3. Bare title similarity ``>= title_threshold`` (default 0.70). + After the version annotation is stripped, the underlying titles + agree. + 4. Artist similarity ``>= artist_threshold`` (default 0.60). Same + artist credit. + + When ``expected_version == matched_version`` returns ``True`` + trivially — no mismatch to decide. + + Examples that ACCEPT (return True): + + - expected=``'live'``, matched=``'original'``, fp=0.95, title=0.95, + artist=1.0 — typical live-recording MB-bare-title case (issue + #607 example 2). + - expected=``'original'``, matched=``'live'`` — same case in the + other direction. + + Examples that REJECT (return False): + + - expected=``'instrumental'``, matched=``'original'`` — fingerprint + matched the instrumental recording; if user asked for vocal, the + file is genuinely wrong. Stays strict regardless of confidence. + - expected=``'remix'``, matched=``'original'`` — same logic. + - expected=``'live'``, matched=``'remix'`` — both versioned, + both different. Real mismatch. + - expected=``'live'``, matched=``'original'``, fp=0.50 — one-sided + live but low confidence. Fall through to FAIL. + - expected=``'live'``, matched=``'original'``, fp=0.95 but title + sim 0.30 — bare titles don't agree. Different song. + """ + if expected_version == matched_version: + return True + + one_sided_live = ( + (expected_version in _LIVE_AWARE_VERSIONS and matched_version == _BARE_VERSION) + or (expected_version == _BARE_VERSION and matched_version in _LIVE_AWARE_VERSIONS) + ) + if not one_sided_live: + return False + + return ( + fingerprint_score >= score_threshold + and title_similarity >= title_threshold + and artist_similarity >= artist_threshold + ) + + +__all__ = ['is_acceptable_version_mismatch'] diff --git a/core/metadata/__init__.py b/core/metadata/__init__.py index e7142adf..ee89ee19 100644 --- a/core/metadata/__init__.py +++ b/core/metadata/__init__.py @@ -31,6 +31,7 @@ from core.metadata.registry import ( get_discogs_client, get_hydrabase_client, get_itunes_client, + get_musicbrainz_client, get_primary_client, get_primary_source, get_spotify_client_for_profile, @@ -82,6 +83,7 @@ __all__ = [ "get_metadata_cache", "get_metadata_source_status", "get_metadata_service", + "get_musicbrainz_client", "get_musicmap_similar_artists", "get_primary_client", "get_primary_source", diff --git a/core/metadata/artist_resolution.py b/core/metadata/artist_resolution.py new file mode 100644 index 00000000..362a6760 --- /dev/null +++ b/core/metadata/artist_resolution.py @@ -0,0 +1,74 @@ +"""Pure artist-list resolution for tag-write paths. + +Single source of truth for "what is the canonical multi-value artists +list for this track?" Different download paths populate `context` with +different keys — Deezer-direct downloads stamp `original_search.artists` +as a proper list, but Soulseek matched downloads only carry `artist` +(singular string) in `original_search_result` while the full list lives +on `track_info` (the full Spotify track object). + +Resolution order: + 1. `context.original_search_result.artists` (preferred — already- + curated by the source path that constructed the context) + 2. `context.track_info.artists` (Spotify/Deezer/Tidal full track + object — always carries the artists array when matched) + 3. `[artist_dict.name]` as a single-element fallback when neither + carries a list (primary-artist-only) + +Each list item may be a dict with a `name` key (Spotify shape), a bare +string, or any other object — the helper normalizes all three to +strings and drops empty entries. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + + +def _normalize_artists_iterable(items: Any) -> List[str]: + if not isinstance(items, list): + return [] + result: List[str] = [] + for item in items: + if isinstance(item, dict): + name = item.get("name") + if isinstance(name, str) and name.strip(): + result.append(name.strip()) + elif isinstance(item, str): + stripped = item.strip() + if stripped: + result.append(stripped) + elif item is not None: + text = str(item).strip() + if text: + result.append(text) + return result + + +def resolve_track_artists( + original_search: Optional[Dict[str, Any]], + track_info: Optional[Dict[str, Any]], + artist_dict: Optional[Dict[str, Any]], +) -> List[str]: + """Return the canonical multi-value artists list for tag-write. + + Falls through preferred → track_info → primary-artist fallback. Each + candidate is normalized to a list of stripped non-empty strings. + Empty list returned only when every candidate is empty/invalid. + """ + if isinstance(original_search, dict): + primary = _normalize_artists_iterable(original_search.get("artists")) + if primary: + return primary + + if isinstance(track_info, dict): + secondary = _normalize_artists_iterable(track_info.get("artists")) + if secondary: + return secondary + + if isinstance(artist_dict, dict): + name = artist_dict.get("name") + if isinstance(name, str) and name.strip(): + return [name.strip()] + + return [] diff --git a/core/metadata/artwork.py b/core/metadata/artwork.py index d405fa01..694933d9 100644 --- a/core/metadata/artwork.py +++ b/core/metadata/artwork.py @@ -28,6 +28,54 @@ __all__ = [ logger = _create_logger("metadata.artwork") +# Query-string keys whose values must be masked when a media-server +# URL ends up in a log line. Plex uses X-Plex-Token, Jellyfin uses +# X-Emby-Token / api_key, Navidrome's Subsonic auth uses t (token) + +# s (salt) + p (password fallback). Logs end up persisted to disk — +# leaking any of these gives full read access to the user's library. +_REDACT_QUERY_KEYS = ( + 'x-plex-token', 'x-emby-token', 'api_key', 'apikey', + 't', 's', 'p', 'token', 'password', +) +_REDACT_KEYS_ALT = '|'.join(re.escape(k) for k in _REDACT_QUERY_KEYS) +# Plain form: `?key=value` or `&key=value`. Anchored on `?` / `&` (or +# string start) so short keys like `t` only match at parameter +# boundaries — not as a substring of `format=Jpg`. +_REDACT_QUERY_RE = re.compile( + r'(?i)(?P^|[?&])(?P' + _REDACT_KEYS_ALT + r')=(?P[^&\s]+)' +) +# URL-encoded form: `%3Fkey%3Dvalue` or `%26key%3Dvalue`. The image +# proxy wraps the original URL via `?url=`, so the auth +# params end up encoded inside another URL. Without this second pass +# the encoded form survives plain redaction and ships to logs intact. +_REDACT_QUERY_RE_ENCODED = re.compile( + r'(?i)(?P%3F|%26)(?P' + _REDACT_KEYS_ALT + r')%3D(?P[^%&\s]+?)(?=%26|&|\s|$)' +) + + +def _redact_url_secrets(url: str | None) -> str: + """Mask sensitive query parameters in a URL so the result is safe + to log. Handles both the plain form (``?token=abc``) and the URL- + encoded form (``%3Ftoken%3Dabc``) — the latter shows up when an + auth-bearing URL is wrapped inside another URL's query string + (e.g. our `/api/image-proxy?url=` flow). + + Returns ``''`` for None/empty input. Idempotent (safe to call on + already-redacted strings).""" + if not url: + return '' + out = str(url) + out = _REDACT_QUERY_RE.sub( + lambda m: f"{m.group('lead')}{m.group('key')}=***REDACTED***", + out, + ) + out = _REDACT_QUERY_RE_ENCODED.sub( + lambda m: f"{m.group('lead')}{m.group('key')}%3D***REDACTED***", + out, + ) + return out + + def normalize_image_url(thumb_url: str | None) -> str | None: """Convert media-server image URLs into browser-safe URLs.""" if not thumb_url: @@ -62,7 +110,6 @@ def normalize_image_url(thumb_url: str | None) -> str | None: plex_config = cfg.get_plex_config() plex_base_url = plex_config.get('base_url', '') plex_token = plex_config.get('token', '') - logger.info("Plex config - base_url: %s, token: %s...", plex_base_url, plex_token[:10]) if plex_base_url and plex_token: # Extract the path from URL @@ -76,18 +123,13 @@ def normalize_image_url(thumb_url: str | None) -> str | None: # Construct proper Plex URL with token fixed_url = f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}" - logger.info("Fixed URL: %s", fixed_url) + logger.debug("Fixed URL: %s", _redact_url_secrets(fixed_url)) return _browser_safe_image_url(fixed_url) elif active_server == 'jellyfin': jellyfin_config = cfg.get_jellyfin_config() jellyfin_base_url = jellyfin_config.get('base_url', '') jellyfin_token = jellyfin_config.get('api_key', '') - logger.info( - "Jellyfin config - base_url: %s, token: %s...", - jellyfin_base_url, - jellyfin_token[:10] if jellyfin_token else 'None', - ) if jellyfin_base_url: # Extract the path from URL @@ -105,7 +147,7 @@ def normalize_image_url(thumb_url: str | None) -> str | None: fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}" else: fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}" - logger.info("Fixed URL: %s", fixed_url) + logger.debug("Fixed URL: %s", _redact_url_secrets(fixed_url)) return _browser_safe_image_url(fixed_url) elif active_server == 'navidrome': @@ -113,7 +155,6 @@ def normalize_image_url(thumb_url: str | None) -> str | None: navidrome_base_url = navidrome_config.get('base_url', '') navidrome_username = navidrome_config.get('username', '') navidrome_password = navidrome_config.get('password', '') - logger.info("Navidrome config - base_url: %s, username: %s", navidrome_base_url, navidrome_username) if navidrome_base_url and navidrome_username and navidrome_password: # Extract the path from URL @@ -137,7 +178,7 @@ def normalize_image_url(thumb_url: str | None) -> str | None: # Construct proper Navidrome Subsonic URL fixed_url = f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}" - logger.info("Fixed URL: %s", fixed_url) + logger.debug("Fixed URL: %s", _redact_url_secrets(fixed_url)) return _browser_safe_image_url(fixed_url) logger.warning("No configuration found for %s or unsupported server type", active_server) @@ -146,18 +187,18 @@ def normalize_image_url(thumb_url: str | None) -> str | None: return _browser_safe_image_url(thumb_url) except Exception as exc: - logger.error("Error fixing image URL '%s': %s", thumb_url, exc) + logger.error("Error fixing image URL '%s': %s", _redact_url_secrets(thumb_url), exc) return _browser_safe_image_url(thumb_url) def is_image_proxy_url(url: str) -> bool: - """Return True for SoulSync image-proxy URLs, absolute or relative.""" + """Return True for SoulSync image proxy/cache URLs, absolute or relative.""" if not url: return False try: parsed = urlparse(url) - return parsed.path == '/api/image-proxy' + return parsed.path == '/api/image-proxy' or parsed.path.startswith('/api/image-cache/') except Exception: return False @@ -194,10 +235,19 @@ def _browser_safe_image_url(url: str) -> str: if is_image_proxy_url(url): return url - if url.startswith('/api/image-proxy?url='): + if url.startswith('/api/image-proxy?url=') or url.startswith('/api/image-cache/'): return url if url.startswith('http://') or url.startswith('https://'): + try: + from core.image_cache import cached_image_url + + cached_url = cached_image_url(url) + if cached_url: + return cached_url + except Exception as exc: + logger.debug("image cache URL registration failed: %s", exc) + if is_internal_image_host(url): return f"/api/image-proxy?url={quote(url, safe='')}" return url diff --git a/core/metadata/cache.py b/core/metadata/cache.py index 2974c5eb..b49daae5 100644 --- a/core/metadata/cache.py +++ b/core/metadata/cache.py @@ -8,6 +8,7 @@ Transparent to callers: check cache before API call, store after success. import json import logging import threading +import time from datetime import datetime from typing import Optional, Dict, List, Tuple @@ -49,6 +50,33 @@ class MetadataCache: from database.music_database import get_database return get_database() + @staticmethod + def _is_transient_sqlite_io_error(exc: Exception) -> bool: + return 'disk i/o error' in str(exc).lower() + + def _run_maintenance_write(self, label: str, operation, default: int = 0) -> int: + """Run a maintenance write with one retry for transient SQLite I/O.""" + for attempt in range(2): + try: + db = self._get_db() + conn = db._get_connection() + try: + return operation(conn) + finally: + conn.close() + except Exception as e: + if self._is_transient_sqlite_io_error(e) and attempt == 0: + logger.warning( + "%s hit transient SQLite disk I/O error; retrying once: %s", + label, + e, + ) + time.sleep(0.25) + continue + logger.error("%s error: %s", label, e) + return default + return default + # ─── Entity Methods ─────────────────────────────────────────────── def get_entity(self, source: str, entity_type: str, entity_id: str) -> Optional[dict]: @@ -566,137 +594,113 @@ class MetadataCache: def evict_expired(self) -> int: """Delete entries that have exceeded their TTL. Returns count of evicted entries.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() + def _operation(conn): + cursor = conn.cursor() - # Entities - cursor.execute(""" - DELETE FROM metadata_cache_entities - WHERE julianday('now') - julianday(updated_at) > ttl_days - """) - entity_count = cursor.rowcount + # Entities + cursor.execute(""" + DELETE FROM metadata_cache_entities + WHERE julianday('now') - julianday(updated_at) > ttl_days + """) + entity_count = cursor.rowcount - # Searches - cursor.execute(""" - DELETE FROM metadata_cache_searches - WHERE julianday('now') - julianday(created_at) > ttl_days - """) - search_count = cursor.rowcount + # Searches + cursor.execute(""" + DELETE FROM metadata_cache_searches + WHERE julianday('now') - julianday(created_at) > ttl_days + """) + search_count = cursor.rowcount - conn.commit() - total = entity_count + search_count - if total > 0: - logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)") - return total - finally: - conn.close() - except Exception as e: - logger.error(f"Cache eviction error: {e}") - return 0 + conn.commit() + total = entity_count + search_count + if total > 0: + logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)") + return total + + return self._run_maintenance_write("Cache eviction", _operation) def clean_junk_entities(self) -> int: """Delete cached entities with empty/placeholder names.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - junk_names = "', '".join(self._JUNK_NAMES - {''}) # exclude empty, handled separately - cursor.execute(f""" - DELETE FROM metadata_cache_entities - WHERE (name IS NULL - OR TRIM(name) = '' - OR LOWER(TRIM(name)) IN ('{junk_names}')) - AND entity_id NOT LIKE '%\\_features' ESCAPE '\\' - AND entity_id NOT LIKE '%\\_tracks' ESCAPE '\\' - """) - count = cursor.rowcount - conn.commit() - if count > 0: - logger.info(f"Cleaned {count} junk entities from cache") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"Junk cleanup error: {e}") - return 0 + def _operation(conn): + cursor = conn.cursor() + junk_names = "', '".join(self._JUNK_NAMES - {''}) # exclude empty, handled separately + cursor.execute(f""" + DELETE FROM metadata_cache_entities + WHERE (name IS NULL + OR TRIM(name) = '' + OR LOWER(TRIM(name)) IN ('{junk_names}')) + AND entity_id NOT LIKE '%\\_features' ESCAPE '\\' + AND entity_id NOT LIKE '%\\_tracks' ESCAPE '\\' + """) + count = cursor.rowcount + conn.commit() + if count > 0: + logger.info(f"Cleaned {count} junk entities from cache") + return count + + return self._run_maintenance_write("Junk cleanup", _operation) def clean_orphaned_searches(self) -> int: """Delete search results where <50% of referenced entities still exist.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute("SELECT id, source, search_type, result_ids FROM metadata_cache_searches") - rows = cursor.fetchall() + def _operation(conn): + cursor = conn.cursor() + cursor.execute("SELECT id, source, search_type, result_ids FROM metadata_cache_searches") + rows = cursor.fetchall() - dead_ids = [] - for row in rows: - try: - result_ids = json.loads(row['result_ids'] or '[]') - except (json.JSONDecodeError, TypeError): - dead_ids.append(row['id']) - continue + dead_ids = [] + for row in rows: + try: + result_ids = json.loads(row['result_ids'] or '[]') + except (json.JSONDecodeError, TypeError): + dead_ids.append(row['id']) + continue - if not result_ids: - dead_ids.append(row['id']) - continue + if not result_ids: + dead_ids.append(row['id']) + continue - # Check how many referenced entities still exist - placeholders = ','.join('?' * len(result_ids)) - cursor.execute(f""" - SELECT COUNT(*) FROM metadata_cache_entities - WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders}) - """, [row['source'], row['search_type']] + list(result_ids)) - found = cursor.fetchone()[0] + # Check how many referenced entities still exist + placeholders = ','.join('?' * len(result_ids)) + cursor.execute(f""" + SELECT COUNT(*) FROM metadata_cache_entities + WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders}) + """, [row['source'], row['search_type']] + list(result_ids)) + found = cursor.fetchone()[0] - if found < len(result_ids) * 0.5: - dead_ids.append(row['id']) + if found < len(result_ids) * 0.5: + dead_ids.append(row['id']) - if dead_ids: - # Delete in chunks to stay under SQLite variable limit - for i in range(0, len(dead_ids), 400): - chunk = dead_ids[i:i + 400] - placeholders = ','.join('?' * len(chunk)) - cursor.execute(f"DELETE FROM metadata_cache_searches WHERE id IN ({placeholders})", chunk) - conn.commit() + if dead_ids: + # Delete in chunks to stay under SQLite variable limit + for i in range(0, len(dead_ids), 400): + chunk = dead_ids[i:i + 400] + placeholders = ','.join('?' * len(chunk)) + cursor.execute(f"DELETE FROM metadata_cache_searches WHERE id IN ({placeholders})", chunk) + conn.commit() - count = len(dead_ids) - if count > 0: - logger.info(f"Cleaned {count} orphaned search results from cache") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"Orphan search cleanup error: {e}") - return 0 + count = len(dead_ids) + if count > 0: + logger.info(f"Cleaned {count} orphaned search results from cache") + return count + + return self._run_maintenance_write("Orphan search cleanup", _operation) def clean_stale_musicbrainz_nulls(self, max_age_days: int = 30) -> int: """Delete MusicBrainz cache entries where lookup found nothing (null MBID) and age > max_age_days.""" - try: - db = self._get_db() - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - DELETE FROM musicbrainz_cache - WHERE musicbrainz_id IS NULL - AND julianday('now') - julianday(last_updated) > ? - """, (max_age_days,)) - count = cursor.rowcount - conn.commit() - if count > 0: - logger.info(f"Cleaned {count} stale MusicBrainz null entries (>{max_age_days} days)") - return count - finally: - conn.close() - except Exception as e: - logger.error(f"MusicBrainz null cleanup error: {e}") - return 0 + def _operation(conn): + cursor = conn.cursor() + cursor.execute(""" + DELETE FROM musicbrainz_cache + WHERE musicbrainz_id IS NULL + AND julianday('now') - julianday(last_updated) > ? + """, (max_age_days,)) + count = cursor.rowcount + conn.commit() + if count > 0: + logger.info(f"Cleaned {count} stale MusicBrainz null entries (>{max_age_days} days)") + return count + + return self._run_maintenance_write("MusicBrainz null cleanup", _operation) def get_health_stats(self) -> dict: """Return cache health statistics for the repair dashboard. diff --git a/core/metadata/completion.py b/core/metadata/completion.py index 97baefb0..dec03161 100644 --- a/core/metadata/completion.py +++ b/core/metadata/completion.py @@ -132,6 +132,7 @@ def check_album_completion( confidence_threshold=0.7, server_source=active_server, candidate_albums=candidate_albums, + strict_discography_match=True, ) except Exception as db_error: logger.error(f"Database error for album '{album_name}': {db_error}") @@ -239,6 +240,7 @@ def check_single_completion( confidence_threshold=0.7, server_source=active_server, candidate_albums=candidate_albums, + strict_discography_match=True, ) except Exception as db_error: logger.error(f"Database error for EP '{single_name}': {db_error}") diff --git a/core/metadata/enrichment.py b/core/metadata/enrichment.py index 051426f8..43e668e8 100644 --- a/core/metadata/enrichment.py +++ b/core/metadata/enrichment.py @@ -102,7 +102,19 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf save_audio_file(audio_file, symbols) return True - track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}" + # Discord report (Netti93) — many album-dict construction + # sites pass `total_tracks: 0` when source data is incomplete + # (per types.py, 0 means "unknown"). Pre-fix this serialized + # to "6/0" tags. Helper drops the `/N` suffix when total is + # unknown so the tag reads "6" instead — matches retag's + # behavior at core/tag_writer.py and ID3 spec convention. + from core.metadata.track_number_format import ( + format_track_number_tag, + format_track_number_tuple, + ) + track_num_str = format_track_number_tag( + metadata.get('track_number'), metadata.get('total_tracks') + ) write_multi = cfg.get("metadata_enhancement.tags.write_multi_artist", False) artists_list = metadata.get("_artists_list", []) @@ -167,7 +179,9 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf audio_file["\xa9day"] = [metadata["date"]] if metadata.get("genre"): audio_file["\xa9gen"] = [metadata["genre"]] - audio_file["trkn"] = [(metadata.get("track_number", 1), metadata.get("total_tracks", 1))] + audio_file["trkn"] = [format_track_number_tuple( + metadata.get("track_number"), metadata.get("total_tracks") + )] if metadata.get("disc_number"): audio_file["disk"] = [(metadata["disc_number"], 0)] diff --git a/core/metadata/registry.py b/core/metadata/registry.py index d2dc86f7..3299d55a 100644 --- a/core/metadata/registry.py +++ b/core/metadata/registry.py @@ -18,13 +18,14 @@ logger = get_logger("metadata.registry") MetadataClientFactory = Callable[[], Any] -METADATA_SOURCE_PRIORITY = ("deezer", "itunes", "spotify", "discogs", "hydrabase") +METADATA_SOURCE_PRIORITY = ("deezer", "itunes", "spotify", "discogs", "hydrabase", "musicbrainz") METADATA_SOURCE_LABELS = { "spotify": "Spotify", "itunes": "iTunes", "deezer": "Deezer", "discogs": "Discogs", "hydrabase": "Hydrabase", + "musicbrainz": "MusicBrainz", } _UNSET = object() @@ -140,6 +141,22 @@ def _get_discogs_factory(client_factory: Optional[MetadataClientFactory]) -> Met return DiscogsClient +def _get_amazon_factory(client_factory: Optional[MetadataClientFactory]) -> MetadataClientFactory: + if client_factory is not None: + return client_factory + from core.amazon_client import AmazonClient + + return AmazonClient + + +def _get_musicbrainz_factory(client_factory: Optional[MetadataClientFactory]) -> MetadataClientFactory: + if client_factory is not None: + return client_factory + from core.musicbrainz_search import MusicBrainzSearchClient + + return MusicBrainzSearchClient + + def get_spotify_client(client_factory: Optional[MetadataClientFactory] = None): """Get shared Spotify client. @@ -260,6 +277,30 @@ def get_discogs_client( return client +def get_amazon_client(client_factory: Optional[MetadataClientFactory] = None): + """Get cached Amazon Music client.""" + cache_key = "amazon" + factory = _get_amazon_factory(client_factory) + with _client_cache_lock: + client = _client_cache.get(cache_key) + if client is None: + client = factory() + _client_cache[cache_key] = client + return client + + +def get_musicbrainz_client(client_factory: Optional[MetadataClientFactory] = None): + """Get cached MusicBrainz primary source client.""" + cache_key = "musicbrainz" + factory = _get_musicbrainz_factory(client_factory) + with _client_cache_lock: + client = _client_cache.get(cache_key) + if client is None: + client = factory() + _client_cache[cache_key] = client + return client + + def is_hydrabase_enabled() -> bool: """Return True when Hydrabase is connected and app-enabled.""" try: @@ -288,24 +329,26 @@ def get_hydrabase_client(allow_fallback: bool = True, require_enabled: bool = Tr def get_primary_source(spotify_client_factory: Optional[MetadataClientFactory] = None) -> str: """Return configured primary metadata source.""" - source = _get_config_value("metadata.fallback_source", "deezer") or "deezer" + _default = METADATA_SOURCE_PRIORITY[0] + source = _get_config_value("metadata.fallback_source", _default) or _default if source == "spotify": try: spotify = get_spotify_client(client_factory=spotify_client_factory) if not spotify or not spotify.is_spotify_authenticated(): - return "deezer" + return _default except Exception: - return "deezer" + return _default return source def get_spotify_disconnect_source(configured_source: Optional[str] = None) -> str: """Return the active metadata source after Spotify is disconnected.""" - source = configured_source if configured_source is not None else _get_config_value("metadata.fallback_source", "deezer") - source = source or "deezer" - return "deezer" if source == "spotify" else source + _default = METADATA_SOURCE_PRIORITY[0] + source = configured_source if configured_source is not None else _get_config_value("metadata.fallback_source", _default) + source = source or _default + return _default if source == "spotify" else source def get_metadata_source_label(source: str) -> str: @@ -331,6 +374,8 @@ def get_primary_client( itunes_client_factory: Optional[MetadataClientFactory] = None, deezer_client_factory: Optional[MetadataClientFactory] = None, discogs_client_factory: Optional[MetadataClientFactory] = None, + amazon_client_factory: Optional[MetadataClientFactory] = None, + musicbrainz_client_factory: Optional[MetadataClientFactory] = None, ): """Return client for configured primary source.""" return get_client_for_source( @@ -339,6 +384,8 @@ def get_primary_client( itunes_client_factory=itunes_client_factory, deezer_client_factory=deezer_client_factory, discogs_client_factory=discogs_client_factory, + amazon_client_factory=amazon_client_factory, + musicbrainz_client_factory=musicbrainz_client_factory, ) @@ -348,6 +395,8 @@ def get_primary_source_status( itunes_client_factory: Optional[MetadataClientFactory] = None, deezer_client_factory: Optional[MetadataClientFactory] = None, discogs_client_factory: Optional[MetadataClientFactory] = None, + amazon_client_factory: Optional[MetadataClientFactory] = None, + musicbrainz_client_factory: Optional[MetadataClientFactory] = None, ) -> Dict[str, Any]: """Return a generic status snapshot for the active primary metadata source.""" source = _get_config_value("metadata.fallback_source", "deezer") or "deezer" @@ -361,6 +410,8 @@ def get_primary_source_status( itunes_client_factory=itunes_client_factory, deezer_client_factory=deezer_client_factory, discogs_client_factory=discogs_client_factory, + amazon_client_factory=amazon_client_factory, + musicbrainz_client_factory=musicbrainz_client_factory, ) if source == "spotify": connected = bool(client and client.is_spotify_authenticated()) @@ -387,6 +438,8 @@ def get_client_for_source( itunes_client_factory: Optional[MetadataClientFactory] = None, deezer_client_factory: Optional[MetadataClientFactory] = None, discogs_client_factory: Optional[MetadataClientFactory] = None, + amazon_client_factory: Optional[MetadataClientFactory] = None, + musicbrainz_client_factory: Optional[MetadataClientFactory] = None, ): """Return exact client for a source, or None if unavailable.""" if source == "spotify": @@ -410,4 +463,10 @@ def get_client_for_source( if source == "itunes": return get_itunes_client(client_factory=itunes_client_factory) + if source == "amazon": + return get_amazon_client(client_factory=amazon_client_factory) + + if source == "musicbrainz": + return get_musicbrainz_client(client_factory=musicbrainz_client_factory) + return None diff --git a/core/metadata/release_type.py b/core/metadata/release_type.py new file mode 100644 index 00000000..1f0b6b9b --- /dev/null +++ b/core/metadata/release_type.py @@ -0,0 +1,93 @@ +"""Canonical mapping from raw provider release-type vocabulary to the +internal `album_type` field that drives discography binning + UI. + +Why this exists +--------------- +Three sites historically duplicated the same "best-effort primary-type +→ album_type" mapping, each with a slightly different vocabulary: + + core/musicbrainz_search.py: `_map_release_type` knew about + {album, single, ep, compilation}, defaulted unknown → 'album'. + core/metadata/types.py: inline `{single: single, ep: ep}.get(...)` — + didn't even know about 'compilation', also defaulted → 'album'. + core/metadata/cache.py: Deezer-specific record_type validator — + intentionally narrow, kept here for its provider. + +Issue #650 (S-Bryce) reported that MusicBrainz tags music videos and +some legitimate singles with primary-type=`Other`, which both mappers +silently routed to `album_type='album'`. Combined with the API-level +filter at `musicbrainz_search.search_albums` (which only requested +`type=album|ep|single` from MB and dropped 'Other' entirely), users +with MB-as-primary saw entire release-groups go missing from artist +discography views, and downloaded tracks from those release-groups +appeared as orphan "ghost" tracks bound to no album card. + +Fix shape: one shared mapper consumed by every provider's +`raw → Album dataclass` projection. Knows about 'other' and +'broadcast' (MB's two remaining primary-type vocabulary words) and +maps them to 'single' so they land in the Singles section of the +artist detail page — they're almost always single-track music +releases (music videos, broadcast singles, one-off web releases). +Falling through to 'album' was the original sin — places them in +Albums view where they look misleading and clutter the proper LP +list. +""" + +from __future__ import annotations + +from typing import List, Optional + + +# MB primary-type vocabulary as of 2026 — `Album | Single | EP | +# Broadcast | Other`. Compilation is a *secondary* type; querying MB +# with type=compilation silently breaks (returns ~10% of expected +# results) — see musicbrainz_client.browse_artist_release_groups docs. +_AUDIO_OTHER_PRIMARY_TYPES = frozenset({'other', 'broadcast'}) + + +def map_release_group_type(primary_type: Optional[str], + secondary_types: Optional[List[str]] = None) -> str: + """Project a raw provider release-group primary-type + secondary-types + into the internal `album_type` value the UI binning expects. + + Returns one of: `'album'`, `'single'`, `'ep'`, `'compilation'`. + + Mapping rules: + + - `single` / `ep` pass through unchanged. + - `compilation` (primary or secondary) becomes `'compilation'`. The + compilation secondary-type check is required because MB's + canonical pattern is `primary=Album, secondary=[Compilation]`. + - `other` / `broadcast` become `'single'`. Almost always + single-track music releases (music videos, one-off web drops, + broadcast singles). Placing them in Singles is the pragmatic + bucket — they're not LPs, but excluding them entirely (the + pre-fix behaviour) hid legitimate tracks. + - Anything else (including empty/None) → `'album'`. Matches the + pre-fix default so no existing classifications shift. + + `secondary_types` is optional because the legacy types.py call site + doesn't have access to it from the same level of raw structure. + Pass `None` (or omit) for the secondary-types-unavailable path. + """ + pt = (primary_type or '').strip().lower() + + if pt == 'single': + return 'single' + if pt == 'ep': + return 'ep' + if pt == 'compilation': + return 'compilation' + + # Secondary-type override: MB's compilation albums always carry + # `primary=Album` with `secondary=[Compilation]`, so the primary + # check above can't catch them. + if secondary_types: + normalized = {str(s).strip().lower() for s in secondary_types if s} + if 'compilation' in normalized: + return 'compilation' + + if pt in _AUDIO_OTHER_PRIMARY_TYPES: + return 'single' + + return 'album' diff --git a/core/metadata/source.py b/core/metadata/source.py index 67350205..65ac780c 100644 --- a/core/metadata/source.py +++ b/core/metadata/source.py @@ -23,6 +23,7 @@ from core.imports.context import ( get_source_tag_names, normalize_import_context, ) +from core.metadata.artist_resolution import resolve_track_artists from core.metadata.registry import get_itunes_client from database.music_database import get_database from core.metadata.common import ( @@ -182,6 +183,27 @@ def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold +def _normalize_release_date_tag(value: Any) -> str: + """Return a tag-safe release date without inventing missing precision.""" + raw = str(value or "").strip() + if not raw: + return "" + + # Source APIs commonly return ISO timestamps. Audio DATE/TDRC tags should + # receive only the date precision the source actually provided. + raw = raw.split("T", 1)[0].strip() + match = re.match(r"^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$", raw) + if not match: + return "" + + year, month, day = match.groups() + if day: + return f"{year}-{month}-{day}" + if month: + return f"{year}-{month}" + return year + + def _collect_source_ids(metadata: dict, cfg) -> dict: source_ids = {} source = (metadata.get("source") or "").strip().lower() @@ -907,17 +929,13 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di else: logger.warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) - artists = original_search.get("artists") - if isinstance(artists, list) and artists: - all_artists = [] - for artist_item in artists: - if isinstance(artist_item, dict) and artist_item.get("name"): - all_artists.append(artist_item["name"]) - elif isinstance(artist_item, str): - all_artists.append(artist_item) - else: - all_artists.append(str(artist_item)) - + # Resolve canonical artists list. Soulseek matched-download contexts + # only carry `original_search.artist` (singular string) — the full + # contributors list lives on `track_info` (the matched Spotify/etc + # track object). Deezer-direct contexts populate `original_search.artists` + # directly. Pure helper handles all three shapes. + all_artists = resolve_track_artists(original_search, track_info, artist_dict) + if all_artists: # Deezer upgrade path: Deezer's `/search` endpoint only returns # the primary artist for each track. The full contributors # array (feat., remix collaborators, producers credited as @@ -1061,7 +1079,9 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di metadata["disc_number"] = disc_num if disc_num is not None else 1 if album_ctx and album_ctx.get("release_date"): - metadata["date"] = album_ctx["release_date"][:4] + release_date = _normalize_release_date_tag(album_ctx.get("release_date")) + if release_date: + metadata["date"] = release_date genres = artist_dict.get("genres") or [] if genres: @@ -1080,11 +1100,12 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di metadata["album_art_url"] = album_image logger.info( - "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", + "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | date=%s | track=%s/%s | disc=%s", metadata.get("title"), metadata.get("artist"), metadata.get("album_artist"), metadata.get("album"), + metadata.get("date", ""), metadata.get("track_number"), metadata.get("total_tracks"), metadata.get("disc_number"), diff --git a/core/metadata/track_number_format.py b/core/metadata/track_number_format.py new file mode 100644 index 00000000..5feb5bfe --- /dev/null +++ b/core/metadata/track_number_format.py @@ -0,0 +1,77 @@ +"""Format track-number tags consistently across audio formats. + +Discord report (Netti93): album tracks were tagged as ``TRCK = "6/0"`` +instead of ``"6/13"``. Cause: many album-dict construction sites in +the codebase pass ``total_tracks: 0`` when the source data is +incomplete, and ``core/metadata/enrichment.py`` formatted the tag +unconditionally as ``f"{track_number}/{total_tracks}"`` — so 0 +propagated straight to disk. The retag path was unaffected because +``core/tag_writer.py`` already does the right thing. + +Per ``core/metadata/types.py``, ``total_tracks = 0`` is documented +as "unknown" — not an actual track count. Fix at the consumer +boundary so every album-dict constructor doesn't need to be touched. + +This module provides one pure helper. Tests at the function boundary. +""" + +from __future__ import annotations + +from typing import Optional, Tuple + + +def format_track_number_tag( + track_number: Optional[int], + total_tracks: Optional[int], +) -> str: + """Return the canonical TRCK / tracknumber tag string. + + - ``track_number=6, total_tracks=13`` → ``"6/13"`` + - ``track_number=6, total_tracks=0`` → ``"6"`` (total unknown) + - ``track_number=6, total_tracks=None`` → ``"6"`` + - ``track_number=None, total_tracks=13`` → ``"1/13"`` (track defaults to 1) + - ``track_number=None, total_tracks=None`` → ``"1"`` + + ID3 spec allows ``TRCK`` to be either ``"N"`` or ``"N/M"``. Vorbis + ``tracknumber`` follows the same convention. Avoiding the ``/0`` + suffix keeps the tag honest — most media servers and taggers + interpret ``6/0`` as "track 6 of 0" which is nonsensical, while + ``6`` reads as "track 6, total unknown". + """ + num = _coerce_positive_int(track_number, default=1) + total = _coerce_positive_int(total_tracks, default=0) + if total > 0: + return f"{num}/{total}" + return str(num) + + +def format_track_number_tuple( + track_number: Optional[int], + total_tracks: Optional[int], +) -> Tuple[int, int]: + """Return the MP4 ``trkn`` tuple ``(track, total)``. + + MP4 tag spec stores track-of as a 2-int tuple — convention is + ``(N, 0)`` when the total is unknown. Same coercion rules as + ``format_track_number_tag``: missing / None / non-positive + ``track_number`` defaults to 1, missing / 0 / negative + ``total_tracks`` returns 0 (the spec's "unknown" marker). + """ + num = _coerce_positive_int(track_number, default=1) + total = _coerce_positive_int(total_tracks, default=0) + return (num, total) + + +def _coerce_positive_int(value, *, default: int) -> int: + """Coerce to a non-negative int. Falls back to ``default`` for + None / non-numeric / negative input. Floats truncate. + """ + if value is None: + return default + try: + coerced = int(value) + except (TypeError, ValueError): + return default + if coerced < 0: + return default + return coerced diff --git a/core/metadata/types.py b/core/metadata/types.py index 00c9749b..b3d5e7f0 100644 --- a/core/metadata/types.py +++ b/core/metadata/types.py @@ -333,7 +333,52 @@ class Album: @classmethod def from_musicbrainz_dict(cls, raw: Dict[str, Any]) -> 'Album': - """MusicBrainz ``/release/{mbid}`` response shape (release, not release-group).""" + """MusicBrainz album shape. + + Accepts both raw ``/release/{mbid}`` responses and the normalized + MusicBrainz search adapter shape used by app-facing metadata clients. + """ + if raw.get('name') and not raw.get('title'): + artists = raw.get('artists') or [] + artist_names = [] + primary_artist_id = '' + for artist in artists: + if isinstance(artist, dict): + name = _str(artist.get('name')) + if name: + artist_names.append(name) + if not primary_artist_id and artist.get('id'): + primary_artist_id = _str(artist['id']) + else: + name = _str(artist) + if name: + artist_names.append(name) + + images = raw.get('images') or [] + image_url = '' + if images and isinstance(images[0], dict): + image_url = _str(images[0].get('url')) + image_url = image_url or _str(raw.get('image_url')) + + external_ids = {} + if raw.get('id'): + external_ids['musicbrainz'] = _str(raw['id']) + + return cls( + id=_str(raw.get('id')), + name=_str(raw.get('name')), + artists=artist_names or ['Unknown Artist'], + release_date=_str(raw.get('release_date')), + total_tracks=_int(raw.get('total_tracks')), + album_type=_str(raw.get('album_type'), default='album') or 'album', + image_url=image_url or None, + artist_id=primary_artist_id or None, + genres=list(raw.get('genres') or []), + source='musicbrainz', + external_ids=external_ids, + external_urls=dict(raw.get('external_urls') or {}), + ) + artist_credit = raw.get('artist-credit') or [] artist_names = [] primary_artist_id = '' @@ -355,10 +400,16 @@ class Album: if raw.get('barcode'): external_ids['barcode'] = _str(raw['barcode']) - # MB `release-group` carries the album-level type (album/single/ep) + # MB `release-group` carries the album-level type (album/single/ep/ + # compilation/other/broadcast). Centralized mapper handles the + # full vocabulary including 'other' / 'broadcast' (issue #650 — + # music videos and one-off releases) so this projection matches + # the search-adapter projection in `core/musicbrainz_search.py`. + from core.metadata.release_type import map_release_group_type rg = raw.get('release-group') or {} - primary_type = _str(rg.get('primary-type'), default='Album').lower() - album_type = {'single': 'single', 'ep': 'ep'}.get(primary_type, 'album') + primary_type = _str(rg.get('primary-type'), default='Album') + secondary_types = rg.get('secondary-types') or [] + album_type = map_release_group_type(primary_type, secondary_types) if rg.get('id'): external_ids['musicbrainz_release_group'] = _str(rg['id']) diff --git a/core/metadata_service.py b/core/metadata_service.py index 5391a80b..a0bdadc1 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -42,8 +42,10 @@ from core.metadata.registry import ( clear_cached_metadata_client, clear_cached_metadata_clients, clear_cached_profile_spotify_client, + get_amazon_client, get_client_for_source, get_deezer_client, + get_musicbrainz_client, get_discogs_client, get_hydrabase_client, get_itunes_client, @@ -75,6 +77,8 @@ except Exception: # pragma: no cover - optional dependency fallback __all__ = [ "METADATA_SOURCE_PRIORITY", + "get_amazon_client", + "get_musicbrainz_client", "MetadataCache", "MetadataLookupOptions", "MetadataProvider", diff --git a/core/musicbrainz_client.py b/core/musicbrainz_client.py index b152f28f..f1d59491 100644 --- a/core/musicbrainz_client.py +++ b/core/musicbrainz_client.py @@ -128,27 +128,45 @@ class MusicBrainzClient: return [] @rate_limited - def search_release(self, album_name: str, artist_name: Optional[str] = None, limit: int = 10) -> List[Dict[str, Any]]: + def search_release(self, album_name: str, artist_name: Optional[str] = None, + limit: int = 10, strict: bool = True) -> List[Dict[str, Any]]: """ - Search for releases (albums) by name - + Search for releases (albums) by name. + Args: album_name: Name of the album to search for artist_name: Optional artist name to narrow search limit: Maximum number of results to return - + strict: When True (default), builds a phrase-match Lucene query + against the `release` and `artist` fields — correct for + enrichment flows where exact name+artist are known. When + False, sends a bare query (album + artist joined) so MB + hits alias / sortname indexes and folds diacritics, + dramatically improving recall for user-facing fuzzy + lookups (e.g. the manual Fix popup). + Returns: List of release results """ try: - # Escape quotes and backslashes for Lucene query - safe_album = album_name.replace('\\', '\\\\').replace('"', '\\"') - query = f'release:"{safe_album}"' - - if artist_name: - safe_artist = artist_name.replace('\\', '\\\\').replace('"', '\\"') - query += f' AND artist:"{safe_artist}"' - + if strict: + # Escape quotes and backslashes for Lucene query + safe_album = album_name.replace('\\', '\\\\').replace('"', '\\"') + query = f'release:"{safe_album}"' + + if artist_name: + safe_artist = artist_name.replace('\\', '\\\\').replace('"', '\\"') + query += f' AND artist:"{safe_artist}"' + else: + # Bare query — MB tokenizes against title + artist credit + + # alias + sortname indexes together with diacritic folding. + # Recovers cases like "Bjork" → "Björk" that strict phrase + # queries miss. + parts = [album_name] + if artist_name: + parts.append(artist_name) + query = ' '.join(p for p in parts if p) + params = { 'query': query, 'fmt': 'json', @@ -173,27 +191,44 @@ class MusicBrainzClient: return [] @rate_limited - def search_recording(self, track_name: str, artist_name: Optional[str] = None, limit: int = 10) -> List[Dict[str, Any]]: + def search_recording(self, track_name: str, artist_name: Optional[str] = None, + limit: int = 10, strict: bool = True) -> List[Dict[str, Any]]: """ - Search for recordings (tracks) by name - + Search for recordings (tracks) by name. + Args: track_name: Name of the track to search for artist_name: Optional artist name to narrow search limit: Maximum number of results to return - + strict: When True (default), builds a phrase-match Lucene query + against the `recording` and `artist` fields — correct for + enrichment flows where exact name+artist are known. When + False, sends a bare query (track + artist joined) so MB + hits alias / sortname indexes and folds diacritics. The + bare path also avoids the AND-clause that kills recall + when either side mis-matches (e.g. "Bjork" vs canonical + "Björk", or a track title with bracketed suffix like + "(Live)" that strict phrase match rejects). + Returns: List of recording results """ try: - # Escape quotes and backslashes for Lucene query - safe_track = track_name.replace('\\', '\\\\').replace('"', '\\"') - query = f'recording:"{safe_track}"' - - if artist_name: - safe_artist = artist_name.replace('\\', '\\\\').replace('"', '\\"') - query += f' AND artist:"{safe_artist}"' - + if strict: + # Escape quotes and backslashes for Lucene query + safe_track = track_name.replace('\\', '\\\\').replace('"', '\\"') + query = f'recording:"{safe_track}"' + + if artist_name: + safe_artist = artist_name.replace('\\', '\\\\').replace('"', '\\"') + query += f' AND artist:"{safe_artist}"' + else: + # Bare query — see search_release for rationale. + parts = [track_name] + if artist_name: + parts.append(artist_name) + query = ' '.join(p for p in parts if p) + params = { 'query': query, 'fmt': 'json', diff --git a/core/musicbrainz_search.py b/core/musicbrainz_search.py index 965317be..f01292d4 100644 --- a/core/musicbrainz_search.py +++ b/core/musicbrainz_search.py @@ -114,18 +114,12 @@ def _extract_title_hint(query: str, artist_name: str) -> Optional[str]: return None -def _map_release_type(primary_type: str, secondary_types: List[str] = None) -> str: - """Map MusicBrainz release group type to standard album_type.""" - pt = (primary_type or '').lower() - if pt == 'album': - return 'album' - elif pt == 'single': - return 'single' - elif pt == 'ep': - return 'ep' - elif pt == 'compilation' or 'compilation' in (secondary_types or []): - return 'compilation' - return 'album' +# Thin module-level alias retained so callers inside this file keep +# working without touching every call site. The canonical implementation +# (including the 'other' / 'broadcast' handling that fixes issue #650) +# lives in `core/metadata/release_type.py` so every provider's `raw → +# Album` projection shares one mapper. +from core.metadata.release_type import map_release_group_type as _map_release_type class MusicBrainzSearchClient: @@ -365,7 +359,15 @@ class MusicBrainzSearchClient: # the filter silently breaks. Actual compilations # (primary-type=Album with secondary-types=[Compilation]) # are handled by the studio-preference filter below. - release_types=['album', 'ep', 'single'], + # 'other' added per issue #650 — MB tags music videos + # and one-off web/broadcast releases with primary=Other, + # and many artists (Vocaloid producers, indie acts, JP + # solo artists) have legitimate singles classified + # there. Pre-fix this filter dropped them at the API + # layer, hiding tracks the user had downloaded. + # `map_release_group_type` routes 'other' into the + # singles bucket so they appear in the right UI section. + release_types=['album', 'ep', 'single', 'other'], limit=100, ) @@ -638,13 +640,34 @@ class MusicBrainzSearchClient: logger.warning(f"MusicBrainz track search failed: {e}") return [] - def _search_tracks_text(self, track_name: str, artist_name: Optional[str], limit: int) -> List[Track]: - """Fallback text-search path for structured/fuzzy track queries.""" + def _search_tracks_text(self, track_name: str, artist_name: Optional[str], limit: int, + strict: bool = True, min_score: Optional[int] = None) -> List[Track]: + """Fallback text-search path for structured/fuzzy track queries. + + `strict=True` (default) keeps the field-scoped Lucene phrase match — + precise enough for enrichment-style flows where the inputs are + already known-clean. `strict=False` switches to a bare-query + MB lookup that hits alias/sortname indexes with diacritic folding — + needed for user-facing fuzzy surfaces (Fix popup cascade) where + recall beats precision because the user picks from the result list. + Mirrors the same toggle already on `search_recording` in + `core/musicbrainz_client.py`. + + `min_score` defaults to `self._MIN_SCORE` (80) — sized for the + enhanced search tab where unfiltered MB results are noisy. Pass + a lower value (or 0) when a downstream stage like + `core.metadata.relevance.rerank_tracks` will re-sort by artist + match — MB's free-text score heavily favours title-text matches + ("Army of Me (Bjork)" cover by HIRS Collective scores 100, + Björk's canonical "Army of Me" scores 28) so a high floor drops + the right answer. + """ try: - results = self._client.search_recording(track_name, artist_name=artist_name, limit=limit) - # Score filter matches the artist/album logic — cuts garbage - # title collisions from unrelated recordings. - results = [r for r in results if (r.get('score', 0) or 0) >= self._MIN_SCORE] + results = self._client.search_recording( + track_name, artist_name=artist_name, limit=limit, strict=strict + ) + threshold = self._MIN_SCORE if min_score is None else min_score + results = [r for r in results if (r.get('score', 0) or 0) >= threshold] tracks = [] for r in results: @@ -656,6 +679,30 @@ class MusicBrainzSearchClient: logger.warning(f"MusicBrainz track search failed: {e}") return [] + def search_tracks_with_artist(self, track: str, artist: str, + limit: int = 10) -> List[Track]: + """Search MB tracks with track + artist passed as separate fields. + + Powers the Fix-popup metadata cascade (`GET /api/musicbrainz/search_tracks`) + and any future surface where the caller already has the title/artist + split and wants the fuzzy-recall MB lookup without going through + `search_tracks`'s structured-query dispatch (`Artist - Track` + splitting, bare-name artist-first browse). + + Uses bare-query mode (`strict=False`) — diacritic-folded, hits + alias/sortname indexes, no `AND`-clause that kills recall when + either side mis-matches. Score floor lowered to 20 (vs the search + tab's 80) so MB recordings whose title doesn't literally contain + the artist name still enter the candidate pool — the endpoint's + `rerank_tracks` pass then sorts by artist-match relevance. Without + this, queries like `Army of Me` + `Bjork` only surface covers + (score 73-100) and miss Björk's canonical recording (score 28). + """ + if not track and not artist: + return [] + return self._search_tracks_text(track, artist or None, limit, + strict=False, min_score=20) + def _pick_representative_release(self, releases: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: """Pick the best release out of a release-group's editions. @@ -678,7 +725,182 @@ class MusicBrainzSearchClient: return sorted(releases, key=_key)[0] - def get_album(self, album_mbid: str) -> Optional[Dict[str, Any]]: + def is_authenticated(self) -> bool: + return True + + def reload_config(self) -> None: + pass + + def get_track_features(self, track_id: str) -> None: + return None + + def get_user_info(self) -> None: + return None + + def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]: + """Return Spotify-compatible track detail dict by recording MBID.""" + try: + rec = self._client.get_recording(track_id, includes=['releases', 'artist-credits', 'release-groups']) + if not rec: + return None + releases = rec.get('releases', []) or [] + releases.sort(key=self._release_preference_key) + first_rel = releases[0] if releases else {} + rg = first_rel.get('release-group', {}) or {} + release_id = first_rel.get('id', '') + rg_id = rg.get('id', '') + image_url = self._cached_art(release_id, rg_id) + artists = _extract_artist_credit(rec.get('artist-credit', [])) + return { + 'id': rec.get('id', ''), + 'name': rec.get('title', ''), + 'artists': [{'name': a, 'id': ''} for a in artists], + 'album': { + 'id': rg_id or release_id, + 'name': first_rel.get('title', ''), + 'images': [{'url': image_url, 'height': 250, 'width': 250}] if image_url else [], + 'release_date': first_rel.get('date') or rg.get('first-release-date') or '', + }, + 'duration_ms': rec.get('length') or 0, + 'track_number': 1, + 'disc_number': 1, + 'preview_url': None, + 'popularity': 0, + 'external_urls': {'musicbrainz': f'https://musicbrainz.org/recording/{track_id}'}, + } + except Exception as e: + logger.error(f'get_track_details({track_id}) error: {e}') + return None + + def get_recording_flat(self, mbid: str) -> Optional[Dict[str, Any]]: + """Return a Fix-popup-compatible flat track dict by recording MBID. + + Distinct from `get_track_details` which returns a Spotify-shaped + nested dict (artists as objects, album as nested object with + images array). The Discovery Fix popup expects the flat shape that + the spotify/deezer/itunes search endpoints produce — artists as a + list of strings, album as a string, single image_url field. + + Used by `GET /api/musicbrainz/recording/` to support the + MBID-paste lookup field — power-user escape hatch when fuzzy auto- + search ranks the wrong recording among many same-title versions. + + Returns None when the MBID is missing or MB returns no recording. + Recording-without-release is valid (album = '', image_url = ''). + """ + if not mbid: + return None + try: + rec = self._client.get_recording( + mbid, includes=['releases', 'artist-credits', 'release-groups'] + ) + if not rec: + return None + + releases = rec.get('releases', []) or [] + releases.sort(key=self._release_preference_key) + first_rel = releases[0] if releases else {} + rg = first_rel.get('release-group', {}) or {} + release_id = first_rel.get('id', '') + rg_id = rg.get('id', '') + + artists = _extract_artist_credit(rec.get('artist-credit', [])) + album_name = first_rel.get('title', '') or '' + image_url = self._cached_art(release_id, rg_id) if (release_id or rg_id) else None + + return { + 'id': rec.get('id', '') or mbid, + 'name': rec.get('title', '') or '', + 'artists': artists if artists else [], + 'album': album_name, + 'duration_ms': rec.get('length') or 0, + 'image_url': image_url or '', + 'external_urls': { + 'musicbrainz': f'https://musicbrainz.org/recording/{mbid}' + }, + } + except Exception as e: + logger.error(f'get_recording_flat({mbid}) error: {e}') + return None + + def get_album_tracks(self, album_mbid: str) -> Optional[Dict[str, Any]]: + """Return {items: [...], total: N} track listing for a release/release-group MBID.""" + album = self.get_album(album_mbid, include_tracks=True) + if album is None: + return None + flat = album.get('tracks', []) + if isinstance(flat, dict): + return flat + return {'items': flat, 'total': len(flat)} + + def get_artist(self, artist_id: str) -> Optional[Dict[str, Any]]: + """Return Spotify-compatible artist detail dict.""" + try: + artist = self._client.get_artist(artist_id, includes=['tags', 'url-rels']) + if not artist: + return None + genres = [t['name'] for t in (artist.get('tags') or []) if isinstance(t, dict) and t.get('name')] + return { + 'id': artist.get('id', artist_id), + 'name': artist.get('name', ''), + 'genres': genres, + 'followers': {'total': 0}, + 'popularity': 0, + 'images': [], + 'external_urls': {'musicbrainz': f'https://musicbrainz.org/artist/{artist_id}'}, + } + except Exception as e: + logger.error(f'get_artist({artist_id}) error: {e}') + return None + + def get_artist_top_tracks(self, artist_id: str, limit: int = 10) -> List[Dict[str, Any]]: + """Return top recordings for an artist, deduplicated by title and sorted by year.""" + try: + recs = self._client.search_recordings_by_artist_mbid(artist_id, limit=100) + for r in recs: + rels = r.get('releases') or [] + if rels: + rels.sort(key=self._release_preference_key) + r['releases'] = rels + studio = [r for r in recs if self._has_studio_release(r)] + recs = studio or recs + seen: set = set() + deduped = [] + for r in recs: + key = (r.get('title') or '').lower().strip() + if not key or key in seen: + continue + seen.add(key) + deduped.append(r) + results = [] + for r in deduped[:limit]: + releases = r.get('releases', []) + first_rel = releases[0] if releases else {} + rg = first_rel.get('release-group', {}) or {} + release_id = first_rel.get('id', '') + rg_id = rg.get('id', '') + artists = _extract_artist_credit(r.get('artist-credit', [])) + image_url = self._cached_art(release_id, rg_id) + results.append({ + 'id': r.get('id', ''), + 'name': r.get('title', ''), + 'artists': [{'name': a, 'id': ''} for a in artists], + 'album': { + 'id': rg_id or release_id, + 'name': first_rel.get('title', ''), + 'images': [{'url': image_url}] if image_url else [], + }, + 'duration_ms': r.get('length') or 0, + 'popularity': 0, + 'preview_url': None, + 'external_urls': {'musicbrainz': f'https://musicbrainz.org/recording/{r.get("id", "")}'}, + }) + return results + except Exception as e: + logger.error(f'get_artist_top_tracks({artist_id}) error: {e}') + return [] + + def get_album(self, album_mbid: str, include_tracks: bool = True) -> Optional[Dict[str, Any]]: """Get full album details with track listing for download modal. The MBID passed in could be either: @@ -713,10 +935,15 @@ class MusicBrainzSearchClient: album['external_urls'] = { 'musicbrainz': f'https://musicbrainz.org/release-group/{album_mbid}' } + if not include_tracks: + album.pop('tracks', None) return album # Path B: release MBID (text-search fallback path) - return self._render_release_as_album(album_mbid) + album = self._render_release_as_album(album_mbid) + if album and not include_tracks: + album.pop('tracks', None) + return album except Exception as e: logger.error(f"MusicBrainz album detail failed for {album_mbid}: {e}") return None @@ -727,6 +954,11 @@ class MusicBrainzSearchClient: shape the download modal expects. `rg_fallback` supplies release-group metadata (type, artist credits) when resolving from a release-group whose releases may be lightly populated.""" + # NOTE: `cover-art-archive` is NOT a valid `inc` param for the + # /release resource — MB returns 400 if you pass it. The CAA flags + # (`{'front': True, 'back': True, ...}`) come back on every release + # response by default, so we read them below without requesting an + # include. release = self._client.get_release( release_mbid, includes=['recordings', 'artist-credits', 'release-groups'] ) @@ -747,7 +979,18 @@ class MusicBrainzSearchClient: album_type = _map_release_type(primary_type, secondary_types) rg_mbid = rg.get('id', '') - image_url = self._cached_art(release_mbid, rg_mbid) + # Use cover-art-archive metadata to pick the right CAA scope. + # release-group scope is preferred (covers all editions), but only + # if the release itself actually has front art — otherwise that URL + # will 404. `cover-art-archive.front` is authoritative with no + # extra network call (returned as part of the release fetch above). + caa = release.get('cover-art-archive') or {} + if caa.get('front'): + image_url = _cover_art_url(release_mbid, scope='release') + elif rg_mbid: + image_url = _cover_art_url(rg_mbid, scope='release-group') + else: + image_url = None tracks = [] total_tracks = 0 @@ -789,7 +1032,7 @@ class MusicBrainzSearchClient: 'external_urls': {'musicbrainz': f'https://musicbrainz.org/release/{release_mbid}'}, } - def get_artist_albums(self, artist_mbid: str, album_type: str = 'album,single') -> List: + def get_artist_albums(self, artist_mbid: str, album_type: str = 'album,single', limit: int = 200) -> List: """Get artist's releases for discography view.""" try: artist = self._client.get_artist(artist_mbid, includes=['release-groups']) @@ -814,7 +1057,7 @@ class MusicBrainzSearchClient: image_url=image_url, external_urls={'musicbrainz': f'https://musicbrainz.org/release-group/{rg_mbid}'}, )) - return albums + return albums[:limit] except Exception as e: logger.warning(f"MusicBrainz artist albums failed: {e}") return [] diff --git a/core/musicbrainz_service.py b/core/musicbrainz_service.py index 20f24023..d9085300 100644 --- a/core/musicbrainz_service.py +++ b/core/musicbrainz_service.py @@ -428,54 +428,52 @@ class MusicBrainzService: return [] # Tier 3: live MB lookup. Search → fetch by MBID → cache. - try: - results = self.mb_client.search_artist(artist_name, limit=3) - except Exception as e: - logger.debug("lookup_artist_aliases: search_artist(%r) raised: %s", artist_name, e) - self._save_to_cache('artist_aliases', artist_name, None, None, {'aliases': []}, 0) - return [] - - if not results: - self._save_to_cache('artist_aliases', artist_name, None, None, {'aliases': []}, 0) - return [] - - # Score each result: combined of name-similarity + MB's own - # relevance. Score range 0.0-1.0. - scored = [] - for result in results: - mb_name = result.get('name', '') - mb_score = result.get('score', 0) - sim = self._calculate_similarity(artist_name, mb_name) - combined = (sim * 0.7) + (mb_score / 100 * 0.3) - mbid = result.get('id') - if mbid: - scored.append((combined, mbid)) + # Issue #586 — strict search queries `artist:"..."` only and + # MISSES alias / sortname indexes. When MB's canonical name is + # the non-Latin form (e.g. `Дмитрий Яблонский`), the user's + # Latin input ("Dmitry Yablonsky") finds nothing under strict. + # Fall back to non-strict (bare query, hits alias + sortname + # indexes) when strict returns empty OR all results fail the + # trust gate. + scored = self._search_and_score_artists(artist_name, strict=True) + if not scored or self._best_score(scored) < 0.85: + non_strict = self._search_and_score_artists(artist_name, strict=False) + if non_strict and (not scored or self._best_score(non_strict) > self._best_score(scored)): + scored = non_strict if not scored: self._save_to_cache('artist_aliases', artist_name, None, None, {'aliases': []}, 0) return [] scored.sort(key=lambda x: -x[0]) - best_score, best_mbid = scored[0] + best_score, best_mbid, best_mb_score = scored[0] - # Strict trust threshold: real matches for distinctive cross- - # script artists (the user-reported case) score >= 0.95. - # Anything below 0.85 is ambiguous and not worth the false- - # positive risk of pulling in aliases for the wrong artist. - if best_score < 0.85: + # Trust gate. Two ways to pass: + # 1. Combined score >= 0.85 (the historical strict bar that + # catches same-script matches) + # 2. MB's OWN score is very high (>= 95) AND the result is + # unambiguous (top result clearly leads). Bridges the + # cross-script case where local similarity is near zero + # ("Dmitry Yablonsky" vs "Дмитрий Яблонский" sim ~0) + # but MB's index found a high-confidence match. + passes_combined = best_score >= 0.85 + passes_mb_only = best_mb_score >= 95 and ( + len(scored) < 2 or (scored[0][2] - scored[1][2]) >= 5 + ) + if not (passes_combined or passes_mb_only): logger.debug( "lookup_artist_aliases: best match for %r below trust " - "threshold (score=%.2f)", artist_name, best_score, + "threshold (combined=%.2f, mb_score=%d)", + artist_name, best_score, best_mb_score, ) self._save_to_cache('artist_aliases', artist_name, None, None, {'aliases': []}, 0) return [] # Ambiguity detection: when 2+ results both score high (within - # 0.1 of the best), the search hit multiple distinct artists - # with similar names ("John Smith" returning 10 different - # John Smiths all at score 100). Pulling aliases for one of - # them could produce wrong matches. Skip + cache empty. - if len(scored) >= 2 and (scored[0][0] - scored[1][0]) < 0.1: + # 0.1 of the best combined), the search hit multiple distinct + # artists with similar names. Pulling aliases for one could + # produce wrong matches. Skip + cache empty. + if len(scored) >= 2 and (scored[0][0] - scored[1][0]) < 0.1 and not passes_mb_only: logger.debug( "lookup_artist_aliases: ambiguous match for %r — top " "two results within 0.1 (%.2f / %.2f). Skipping alias lookup.", @@ -491,6 +489,40 @@ class MusicBrainzService: ) return aliases + def _search_and_score_artists(self, artist_name: str, strict: bool): + """Search MB for an artist and score each result. + + Returns a list of (combined_score, mbid, raw_mb_score) tuples. + Combined score: 70% local similarity + 30% MB's own relevance + score (0..1). raw_mb_score preserved separately so the trust + gate can prefer high-MB-score results in cross-script cases + where local similarity is near zero. + + Returns empty list on any failure. + """ + try: + results = self.mb_client.search_artist(artist_name, limit=3, strict=strict) + except Exception as e: + logger.debug( + "lookup_artist_aliases: search_artist(%r, strict=%s) raised: %s", + artist_name, strict, e, + ) + return [] + scored = [] + for result in results or []: + mb_name = result.get('name', '') + mb_score = result.get('score', 0) + sim = self._calculate_similarity(artist_name, mb_name) + combined = (sim * 0.7) + (mb_score / 100 * 0.3) + mbid = result.get('id') + if mbid: + scored.append((combined, mbid, mb_score)) + return scored + + @staticmethod + def _best_score(scored): + return max((s[0] for s in scored), default=0.0) if scored else 0.0 + def fetch_artist_aliases(self, mbid: str) -> list: """Fetch the alias list for an artist from MusicBrainz. @@ -499,6 +531,14 @@ class MusicBrainzService: artist record. Pull them so SoulSync can recognise that `澤野弘之` and `Hiroyuki Sawano` refer to the same artist. + Issue #586 — for some artists MB's CANONICAL `name` is the + non-Latin spelling (e.g. `Дмитрий Яблонский`) while the + Latin spelling lives in `aliases` — but the inverse also + happens, where the Latin canonical name has the Cyrillic in + aliases. Either way the canonical `name` and `sort-name` are + themselves valid alternate spellings for matching purposes, + so include them alongside the explicit alias entries. + Returns the deduplicated list of alias `name` strings. Returns empty list (NOT None) on any failure — caller should treat empty as "no aliases available, fall back to direct match" so @@ -514,24 +554,38 @@ class MusicBrainzService: return [] if not data: return [] - raw_aliases = data.get('aliases') or [] + + seen = set() + cleaned = [] + + def _add(value): + if not isinstance(value, str): + return + text = value.strip() + if not text: + return + key = text.lower() + if key in seen: + return + seen.add(key) + cleaned.append(text) + + # Canonical name + sort-name treated as aliases for matching — + # they're the strongest cross-script bridge when MB's + # canonical spelling differs from the user's input. + _add(data.get('name')) + _add(data.get('sort-name')) + # MB returns each alias as a dict with `name`, `sort-name`, # `locale`, `primary`, `type`, etc. We only care about the # display name — that's what `actual` artist strings will - # match against. - seen = set() - cleaned = [] - for entry in raw_aliases: + # match against. Also pull alias sort-name when present + # (some entries have a different sortable form). + for entry in data.get('aliases') or []: if not isinstance(entry, dict): continue - name = (entry.get('name') or '').strip() - if not name: - continue - key = name.lower() - if key in seen: - continue - seen.add(key) - cleaned.append(name) + _add(entry.get('name')) + _add(entry.get('sort-name')) return cleaned def update_artist_aliases(self, artist_id: int, aliases: list) -> None: diff --git a/core/personalized/__init__.py b/core/personalized/__init__.py new file mode 100644 index 00000000..18190e61 --- /dev/null +++ b/core/personalized/__init__.py @@ -0,0 +1,24 @@ +"""Standardized personalized-playlist subsystem. + +Replaces the patchwork of `PersonalizedPlaylistsService` (computed-on- +demand views, no persistence) + `discovery_curated_playlists` (ID-only +storage) + `curated_seasonal_playlists` (full storage) with a single +unified abstraction: + +- ``manager.PersonalizedPlaylistManager`` — owns the storage layer + + generator dispatch + refresh lifecycle. +- ``specs.PlaylistKindSpec`` — one spec per playlist KIND + (``hidden_gems``, ``time_machine``, ``seasonal_mix``, etc.) with + generator function, default config, variant resolver, and display- + name template. +- ``types.Track`` / ``types.PlaylistConfig`` — shared dataclasses. + +The legacy ``PersonalizedPlaylistsService`` keeps its existing +generator implementations — they're called BY the manager rather than +duplicated. This means: +- The improved diversity logic / popularity thresholds / blacklist + filtering all stays. +- New behavior layered on top: persistence, refresh-on-demand, + per-playlist user-tweakable config, staleness windows, listening- + history cross-reference, seeded randomization. +""" diff --git a/core/personalized/api.py b/core/personalized/api.py new file mode 100644 index 00000000..bfa9de60 --- /dev/null +++ b/core/personalized/api.py @@ -0,0 +1,163 @@ +"""HTTP endpoint handlers for the personalized-playlists subsystem. + +Wired into the Flask app from web_server.py. Each handler is a thin +wrapper that: +1. Pulls profile id + manager from request context. +2. Calls one PersonalizedPlaylistManager method. +3. Returns a JSON-serializable shape. + +Live routes (registered against the main Flask app): +- GET /api/personalized/playlists — list +- GET /api/personalized/kinds — registry +- GET /api/personalized/playlist/ — singleton +- GET /api/personalized/playlist// — variant +- POST /api/personalized/playlist//refresh — singleton +- POST /api/personalized/playlist///refresh — variant +- PUT /api/personalized/playlist//config — singleton +- PUT /api/personalized/playlist///config — variant + +The handlers themselves are pure functions returning Python dicts so +they're testable without spinning up Flask. The wiring step in +web_server.py wraps them in `jsonify` + URL routing. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from core.personalized.manager import PersonalizedPlaylistManager +from core.personalized.specs import PlaylistKindRegistry, get_registry +from core.personalized.types import PlaylistRecord, Track + + +def _record_to_dict(record: PlaylistRecord) -> Dict[str, Any]: + return { + 'id': record.id, + 'profile_id': record.profile_id, + 'kind': record.kind, + 'variant': record.variant, + 'name': record.name, + 'config': record.config.to_json_dict(), + 'track_count': record.track_count, + 'last_generated_at': record.last_generated_at, + 'last_synced_at': record.last_synced_at, + 'last_generation_source': record.last_generation_source, + 'last_generation_error': record.last_generation_error, + } + + +def _track_to_dict(track: Track) -> Dict[str, Any]: + return { + 'spotify_track_id': track.spotify_track_id, + 'itunes_track_id': track.itunes_track_id, + 'deezer_track_id': track.deezer_track_id, + 'track_name': track.track_name, + 'artist_name': track.artist_name, + 'album_name': track.album_name, + 'album_cover_url': track.album_cover_url, + 'duration_ms': track.duration_ms, + 'popularity': track.popularity, + 'track_data_json': track.track_data_json, + 'source': track.source, + } + + +def list_kinds( + registry: Optional[PlaylistKindRegistry] = None, + manager: Optional[PersonalizedPlaylistManager] = None, +) -> Dict[str, Any]: + """Return every registered playlist kind with metadata. + + UI uses this to render the "available playlists" picker. Each + kind reports whether it requires a variant; when a manager is + supplied AND the kind has a variant_resolver, the resolved + variant list is also included so the UI can render variant + checkboxes without a second round-trip per kind.""" + reg = registry or get_registry() + out = [] + for spec in reg.all(): + entry = { + 'kind': spec.kind, + 'name_template': spec.name_template, + 'description': spec.description, + 'requires_variant': spec.requires_variant, + 'tags': list(spec.tags), + 'default_config': spec.default_config.to_json_dict(), + 'variants': [], + } + if manager is not None and spec.variant_resolver is not None: + try: + entry['variants'] = list(spec.variant_resolver(manager.deps) or []) + except Exception: + entry['variants'] = [] + out.append(entry) + return {'success': True, 'kinds': out} + + +def list_playlists(manager: PersonalizedPlaylistManager, profile_id: int) -> Dict[str, Any]: + """List every persisted playlist for a profile.""" + records = manager.list_playlists(profile_id) + return { + 'success': True, + 'playlists': [_record_to_dict(r) for r in records], + } + + +def get_playlist_with_tracks( + manager: PersonalizedPlaylistManager, + kind: str, + variant: str, + profile_id: int, +) -> Dict[str, Any]: + """Get the playlist row + its current track snapshot. Auto-creates + the row from default config if it doesn't exist (so the UI's first- + paint of an unseen kind works without a separate ensure call).""" + record = manager.ensure_playlist(kind, variant, profile_id) + tracks = manager.get_playlist_tracks(record.id) + return { + 'success': True, + 'playlist': _record_to_dict(record), + 'tracks': [_track_to_dict(t) for t in tracks], + } + + +def refresh_playlist( + manager: PersonalizedPlaylistManager, + kind: str, + variant: str, + profile_id: int, + config_overrides: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Run the kind's generator and persist the snapshot. Returns the + fresh row + tracks.""" + record = manager.refresh_playlist(kind, variant, profile_id, config_overrides=config_overrides) + tracks = manager.get_playlist_tracks(record.id) + return { + 'success': True, + 'playlist': _record_to_dict(record), + 'tracks': [_track_to_dict(t) for t in tracks], + } + + +def update_config( + manager: PersonalizedPlaylistManager, + kind: str, + variant: str, + profile_id: int, + overrides: Dict[str, Any], +) -> Dict[str, Any]: + """Patch the playlist's config with the provided fields.""" + record = manager.update_config(kind, variant, profile_id, overrides) + return { + 'success': True, + 'playlist': _record_to_dict(record), + } + + +__all__ = [ + 'list_kinds', + 'list_playlists', + 'get_playlist_with_tracks', + 'refresh_playlist', + 'update_config', +] diff --git a/core/personalized/generators/__init__.py b/core/personalized/generators/__init__.py new file mode 100644 index 00000000..8bd6309a --- /dev/null +++ b/core/personalized/generators/__init__.py @@ -0,0 +1,27 @@ +"""Per-kind generators for the personalized-playlists subsystem. + +Each module in this subpackage: +1. Defines a generator function ``generate(deps, variant, config)`` + that returns a ``List[Track]``. +2. Calls ``get_registry().register(spec)`` at import time so the + manager auto-discovers it. + +The legacy ``core.personalized_playlists.PersonalizedPlaylistsService`` +keeps its existing implementations — the wrappers in this package +just adapt the call surface (`PlaylistConfig` → method kwargs) and +coerce results into ``Track`` instances. + +To register every generator, import this package — `from +core.personalized import generators` — typically done once at +application startup.""" + +# Importing each module triggers its registration side-effect. +from core.personalized.generators import hidden_gems # noqa: F401 +from core.personalized.generators import discovery_shuffle # noqa: F401 +from core.personalized.generators import popular_picks # noqa: F401 +from core.personalized.generators import time_machine # noqa: F401 +from core.personalized.generators import genre_playlist # noqa: F401 +from core.personalized.generators import daily_mix # noqa: F401 +from core.personalized.generators import fresh_tape # noqa: F401 +from core.personalized.generators import archives # noqa: F401 +from core.personalized.generators import seasonal_mix # noqa: F401 diff --git a/core/personalized/generators/_common.py b/core/personalized/generators/_common.py new file mode 100644 index 00000000..1b5d610f --- /dev/null +++ b/core/personalized/generators/_common.py @@ -0,0 +1,37 @@ +"""Shared helpers for personalized-playlist generators. + +Each per-kind generator module is small + mechanical — it pulls the +legacy ``PersonalizedPlaylistsService`` instance off the deps object +and calls the matching method, then coerces results. This module +holds the bits every generator reuses so we don't repeat them +five times.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.types import Track + + +def get_service(deps: Any): + """Pull the ``PersonalizedPlaylistsService`` instance from deps. + + Generators access the service via ``deps.service``. Tests can + pass a fake deps namespace with a ``service`` attribute that + returns a stub. Raises a clear error if the dep isn't wired.""" + service = getattr(deps, 'service', None) or (deps.get('service') if isinstance(deps, dict) else None) + if service is None: + raise RuntimeError( + "Personalized generator deps missing `service` " + "(PersonalizedPlaylistsService instance). Wire it during " + "PersonalizedPlaylistManager construction." + ) + return service + + +def coerce_tracks(rows: List[dict]) -> List[Track]: + """Convert legacy generator output (list of dicts) into Track + instances. Tolerates None / non-list inputs by returning [].""" + if not rows: + return [] + return [Track.from_dict(row) for row in rows if isinstance(row, dict)] diff --git a/core/personalized/generators/archives.py b/core/personalized/generators/archives.py new file mode 100644 index 00000000..9da9a2f3 --- /dev/null +++ b/core/personalized/generators/archives.py @@ -0,0 +1,35 @@ +"""The Archives (Spotify Discover Weekly) generator. + +Same shape as Fresh Tape — read curated track-id list from +``discovery_curated_playlists`` under ``discovery_weekly_`` +(fallback ``discovery_weekly``), hydrate via discovery pool.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators.fresh_tape import _hydrate_curated +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'archives' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + return _hydrate_curated(deps, 'discovery_weekly', config) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='The Archives', + description='Your Spotify Discover Weekly — curated discovery picks.', + default_config=PlaylistConfig(limit=50, max_per_album=5, max_per_artist=10), + generator=generate, + requires_variant=False, + tags=['curated', 'spotify'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/daily_mix.py b/core/personalized/generators/daily_mix.py new file mode 100644 index 00000000..c3c96e57 --- /dev/null +++ b/core/personalized/generators/daily_mix.py @@ -0,0 +1,83 @@ +"""Daily Mix generator — top library genre → discovery picks. + +Variant = rank position as a string ('1' / '2' / '3' / '4'). Each +mix tracks the user's Nth top library genre and returns discovery +picks within it. Top genres recompute at refresh time, so as the +library evolves a mix's underlying genre can shift -- the playlist +metadata records which genre was used at the most recent refresh +so the UI can label the mix accurately. + +Note: previously this kind ambitiously promised 50% library + 50% +discovery. The library half was a stub (`tracks` table has no +source IDs to sync), so the new generator is discovery-only. +A future enhancement can backfill source IDs into library rows +and re-add the hybrid behavior.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'daily_mix' + +# Default rank set — UI surfaces 4 daily mixes by default. +_DEFAULT_RANKS = ('1', '2', '3', '4') + +# Number of top library genres to consider when ranking. +_MAX_TOP_GENRES = 8 + + +def _resolve_genre_for_rank(service, rank: int) -> str: + """Look up the user's Nth-ranked top library genre. Returns the + genre key or '' when no genre at that rank. + + Calls ``service.get_top_genres_from_library(limit=...)`` and + indexes the resulting (genre, count) tuples by 0-based rank. + """ + top = service.get_top_genres_from_library(limit=_MAX_TOP_GENRES) or [] + if rank < 1 or rank > len(top): + return '' + pair = top[rank - 1] + if not pair: + return '' + # `top` is List[Tuple[str, int]] per service signature. + return pair[0] if isinstance(pair, (tuple, list)) else str(pair) + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + try: + rank = int(variant) + except (TypeError, ValueError) as exc: + raise ValueError(f"Daily Mix variant {variant!r} must be a rank int") from exc + genre = _resolve_genre_for_rank(service, rank) + if not genre: + # User's library doesn't have enough genres for this rank. + return [] + rows = service.get_genre_playlist(genre=genre, limit=config.limit) + return coerce_tracks(rows) + + +def variant_resolver(deps: Any) -> List[str]: + """Return the standard rank set.""" + return list(_DEFAULT_RANKS) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Daily Mix {variant}', + description='Personalized mix based on your top library genres. One mix per top genre rank.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=3), + generator=generate, + variant_resolver=variant_resolver, + requires_variant=True, + tags=['discovery', 'personalized'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/discovery_shuffle.py b/core/personalized/generators/discovery_shuffle.py new file mode 100644 index 00000000..daf13509 --- /dev/null +++ b/core/personalized/generators/discovery_shuffle.py @@ -0,0 +1,33 @@ +"""Discovery Shuffle generator — pure-random discovery pool exploration.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'discovery_shuffle' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + rows = service.get_discovery_shuffle(limit=config.limit) + return coerce_tracks(rows) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Discovery Shuffle', + description='Pure random shuffle from the discovery pool — different every refresh.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=2), + generator=generate, + requires_variant=False, + tags=['discovery'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/fresh_tape.py b/core/personalized/generators/fresh_tape.py new file mode 100644 index 00000000..c179bf6d --- /dev/null +++ b/core/personalized/generators/fresh_tape.py @@ -0,0 +1,119 @@ +"""Fresh Tape (Spotify Release Radar) generator. + +Reads the curated track-id list cached in ``discovery_curated_playlists`` +under ``release_radar_`` (with fallback to ``release_radar``) +and hydrates each ID against the discovery pool to produce full Track +records. The Spotify enrichment worker is responsible for keeping the +curated list fresh — this generator is just a read-and-hydrate path.""" + +from __future__ import annotations + +import json +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'fresh_tape' + + +def _hydrate_curated(deps: Any, curated_type_prefix: str, config: PlaylistConfig) -> List[Track]: + """Shared body for Fresh Tape + Archives — pulls the cached IDs + from discovery_curated_playlists and hydrates them via the live + discovery pool. Returns a Track list trimmed to ``config.limit``.""" + # Allow tests to inject a fake db / service; production flow gets + # them from the manager's deps. + db = getattr(deps, 'database', None) or (deps.get('database') if isinstance(deps, dict) else None) + if db is None: + raise RuntimeError("Curated-playlist generator deps missing `database`") + + profile_id = _resolve_profile_id(deps) + active_source = _resolve_active_source(deps) + + # Try source-specific then generic, mirrors web_server endpoint behavior. + curated_ids = ( + db.get_curated_playlist(f'{curated_type_prefix}_{active_source}', profile_id=profile_id) + or db.get_curated_playlist(curated_type_prefix, profile_id=profile_id) + or [] + ) + if not curated_ids: + return [] + + pool_rows = db.get_discovery_pool_tracks( + limit=5000, new_releases_only=False, + source=active_source, profile_id=profile_id, + ) + by_id = {} + for t in pool_rows: + if active_source == 'spotify' and getattr(t, 'spotify_track_id', None): + by_id[t.spotify_track_id] = t + elif active_source == 'deezer' and getattr(t, 'deezer_track_id', None): + by_id[t.deezer_track_id] = t + elif active_source == 'itunes' and getattr(t, 'itunes_track_id', None): + by_id[t.itunes_track_id] = t + + tracks: List[Track] = [] + for tid in curated_ids: + candidate = by_id.get(tid) + if candidate is None: + continue + # The pool track is a row-like object; coerce to dict for + # Track.from_dict's existing tolerance. + td = getattr(candidate, 'track_data_json', None) + if isinstance(td, str): + try: + td = json.loads(td) + except (ValueError, TypeError): + td = None + track_dict = { + 'spotify_track_id': getattr(candidate, 'spotify_track_id', None), + 'itunes_track_id': getattr(candidate, 'itunes_track_id', None), + 'deezer_track_id': getattr(candidate, 'deezer_track_id', None), + 'track_name': getattr(candidate, 'track_name', ''), + 'artist_name': getattr(candidate, 'artist_name', ''), + 'album_name': getattr(candidate, 'album_name', ''), + 'album_cover_url': getattr(candidate, 'album_cover_url', None), + 'duration_ms': getattr(candidate, 'duration_ms', 0), + 'popularity': getattr(candidate, 'popularity', 0), + 'track_data_json': td, + 'source': getattr(candidate, 'source', active_source), + } + tracks.append(Track.from_dict(track_dict)) + if len(tracks) >= config.limit: + break + return tracks + + +def _resolve_profile_id(deps: Any) -> int: + fn = getattr(deps, 'get_current_profile_id', None) or ( + deps.get('get_current_profile_id') if isinstance(deps, dict) else None + ) + return fn() if callable(fn) else 1 + + +def _resolve_active_source(deps: Any) -> str: + fn = getattr(deps, 'get_active_discovery_source', None) or ( + deps.get('get_active_discovery_source') if isinstance(deps, dict) else None + ) + return fn() if callable(fn) else 'spotify' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + return _hydrate_curated(deps, 'release_radar', config) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Fresh Tape', + description='Your Spotify Release Radar — new releases from artists you follow.', + default_config=PlaylistConfig(limit=50, max_per_album=5, max_per_artist=10), + generator=generate, + requires_variant=False, + tags=['curated', 'spotify'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/genre_playlist.py b/core/personalized/generators/genre_playlist.py new file mode 100644 index 00000000..d74b1f94 --- /dev/null +++ b/core/personalized/generators/genre_playlist.py @@ -0,0 +1,78 @@ +"""Genre Playlist generator — discovery picks within one genre. + +Variant = either a parent-genre key from +``PersonalizedPlaylistsService.GENRE_MAPPING`` (e.g. ``'rock'``, +``'electronic_dance'``) or a specific child-genre keyword (e.g. +``'house'``). Stored variant is always normalized to lowercase +underscore-separated form so the UI and storage agree. +""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'genre_playlist' + + +def _normalize_variant_to_genre_key(variant: str, service) -> str: + """Resolve a variant string back into the genre identifier the + legacy service expects. + + Service accepts both parent-genre KEYS from GENRE_MAPPING (e.g. + 'Electronic/Dance', 'Hip Hop/Rap') and free-form keywords. + The URL-safe variant we store is the parent key with `/` replaced + by `_` and lowercased — e.g. 'electronic_dance'. This helper + inverts that mapping.""" + if not variant: + raise ValueError('Genre playlist requires a variant') + + # Build a once-computed lookup of normalized → original parent key. + mapping = getattr(service, 'GENRE_MAPPING', {}) + for parent_key in mapping.keys(): + normalized = parent_key.lower().replace('/', '_').replace(' ', '_') + if normalized == variant.lower(): + return parent_key + + # Fall through: treat the variant as a free-form keyword (the + # legacy service handles partial matching for those). + return variant + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + genre_key = _normalize_variant_to_genre_key(variant, service) + rows = service.get_genre_playlist(genre=genre_key, limit=config.limit) + return coerce_tracks(rows) + + +def variant_resolver(deps: Any) -> List[str]: + """Return the URL-safe variant for every parent genre defined on + the service. Specific (free-form) genre variants aren't enumerated + — they're created on demand when the user requests a custom one.""" + service = get_service(deps) + mapping = getattr(service, 'GENRE_MAPPING', {}) + return [ + parent.lower().replace('/', '_').replace(' ', '_') + for parent in mapping.keys() + ] + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Genre — {variant}', + description='Discovery picks within one genre. Supports parent-genre families + free-form genre keywords.', + default_config=PlaylistConfig(limit=50, max_per_album=3, max_per_artist=5), + generator=generate, + variant_resolver=variant_resolver, + requires_variant=True, + tags=['genre'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/hidden_gems.py b/core/personalized/generators/hidden_gems.py new file mode 100644 index 00000000..5a5f79de --- /dev/null +++ b/core/personalized/generators/hidden_gems.py @@ -0,0 +1,42 @@ +"""Hidden Gems generator — low-popularity tracks from discovery pool. + +Wraps ``PersonalizedPlaylistsService.get_hidden_gems`` so the +existing source-aware popularity threshold + diversity filter +behavior is preserved verbatim. The user-tweakable knobs that +arrive via ``PlaylistConfig`` (limit) flow through; future config +options (popularity_max override, exclude_recent_days) get layered +on the wrapper without changing the legacy implementation.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'hidden_gems' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + rows = service.get_hidden_gems(limit=config.limit) + return coerce_tracks(rows) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Hidden Gems', + description='Low-popularity discovery picks — underground / indie tracks you probably haven\'t heard.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=3), + generator=generate, + requires_variant=False, + tags=['discovery'], +) + + +# Register at import time so the manager auto-discovers this kind. +# Re-import (e.g. test reloads) is tolerated: only register if absent. +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/popular_picks.py b/core/personalized/generators/popular_picks.py new file mode 100644 index 00000000..c95fa16f --- /dev/null +++ b/core/personalized/generators/popular_picks.py @@ -0,0 +1,33 @@ +"""Popular Picks generator — high-popularity discovery pool picks.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'popular_picks' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + rows = service.get_popular_picks(limit=config.limit) + return coerce_tracks(rows) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Popular Picks', + description='High-popularity tracks from the discovery pool — what most people are listening to.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=3), + generator=generate, + requires_variant=False, + tags=['discovery'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/seasonal_mix.py b/core/personalized/generators/seasonal_mix.py new file mode 100644 index 00000000..0d09853b --- /dev/null +++ b/core/personalized/generators/seasonal_mix.py @@ -0,0 +1,137 @@ +"""Seasonal Mix generator (variant = season key). + +Variant = season key from ``SEASONAL_CONFIG`` (``'halloween'`` / +``'christmas'`` / ``'valentines'`` / ``'summer'`` / ``'spring'`` / +``'autumn'``). One playlist per season — user picks which seasons +to enable; idle seasons can stay un-refreshed until their active +period. + +Reads curated track IDs from ``curated_seasonal_playlists`` (via +``SeasonalDiscoveryService.get_curated_seasonal_playlist``) and +hydrates them against ``seasonal_tracks`` (which carries full +metadata including ``track_data_json`` for sync-ready downstream +use).""" + +from __future__ import annotations + +import json +from typing import Any, List + +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'seasonal_mix' + + +def _resolve_seasonal_service(deps: Any): + """Pull the SeasonalDiscoveryService instance from deps.""" + svc = getattr(deps, 'seasonal_service', None) or ( + deps.get('seasonal_service') if isinstance(deps, dict) else None + ) + if svc is None: + raise RuntimeError( + "Seasonal mix generator deps missing `seasonal_service` " + "(SeasonalDiscoveryService instance)." + ) + return svc + + +def _resolve_database(deps: Any): + db = getattr(deps, 'database', None) or ( + deps.get('database') if isinstance(deps, dict) else None + ) + if db is None: + raise RuntimeError("Seasonal mix generator deps missing `database`") + return db + + +def _resolve_active_source(deps: Any) -> str: + fn = getattr(deps, 'get_active_discovery_source', None) or ( + deps.get('get_active_discovery_source') if isinstance(deps, dict) else None + ) + return fn() if callable(fn) else 'spotify' + + +def _hydrate_seasonal_tracks(db, season_key: str, source: str, track_ids: List[str]) -> List[Track]: + """Look up the seasonal_tracks rows for the given IDs.""" + if not track_ids: + return [] + placeholders = ','.join('?' * len(track_ids)) + with db._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + f""" + SELECT spotify_track_id, track_name, artist_name, album_name, + album_cover_url, duration_ms, popularity, track_data_json + FROM seasonal_tracks + WHERE season_key = ? AND source = ? + AND spotify_track_id IN ({placeholders}) + """, + (season_key, source, *track_ids), + ) + rows = cursor.fetchall() + + by_id = {} + for r in rows: + if hasattr(r, 'keys'): + r = dict(r) + else: + r = dict(zip( + ('spotify_track_id', 'track_name', 'artist_name', 'album_name', + 'album_cover_url', 'duration_ms', 'popularity', 'track_data_json'), + r, + strict=False, + )) + td = r.get('track_data_json') + if isinstance(td, str): + try: + td = json.loads(td) + except (ValueError, TypeError): + td = None + r['track_data_json'] = td + r['source'] = source + by_id[r['spotify_track_id']] = r + + # Preserve curated order. + return [ + Track.from_dict(by_id[tid]) + for tid in track_ids + if tid in by_id + ] + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + if not variant: + raise ValueError('Seasonal Mix requires a season variant') + seasonal_service = _resolve_seasonal_service(deps) + db = _resolve_database(deps) + source = _resolve_active_source(deps) + track_ids = seasonal_service.get_curated_seasonal_playlist(variant, source=source) or [] + tracks = _hydrate_seasonal_tracks(db, variant, source, track_ids) + return tracks[:config.limit] + + +def variant_resolver(deps: Any) -> List[str]: + """Return every season key from SEASONAL_CONFIG.""" + try: + from core.seasonal_discovery import SEASONAL_CONFIG + except Exception: + return [] + return list(SEASONAL_CONFIG.keys()) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Seasonal — {variant}', + description='Holiday / season-themed picks. One playlist per season; user enables which to track.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=3), + generator=generate, + variant_resolver=variant_resolver, + requires_variant=True, + tags=['curated', 'seasonal'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/time_machine.py b/core/personalized/generators/time_machine.py new file mode 100644 index 00000000..b139be7f --- /dev/null +++ b/core/personalized/generators/time_machine.py @@ -0,0 +1,66 @@ +"""Time Machine generator — by-decade discovery picks. + +Variant = decade label like ``'1980s'`` / ``'1990s'`` / ``'2000s'``. +The variant resolver returns the standard decade set; users see one +playlist per decade (each independently configurable / refreshable). +""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'time_machine' + + +# Standard decades the UI exposes. Adjust here when adding eras. +_DEFAULT_DECADES = ('1960s', '1970s', '1980s', '1990s', '2000s', '2010s', '2020s') + + +def _decade_to_year(variant: str) -> int: + """'1980s' -> 1980. Tolerates ' 1980 ', '1980'. + + Raises ValueError for anything that doesn't look like a decade + label so the manager surfaces a clear error instead of generating + garbage.""" + cleaned = (variant or '').strip().rstrip('sS') + try: + year = int(cleaned) + except ValueError as exc: + raise ValueError(f"Time Machine variant {variant!r} not a decade label") from exc + if year < 1900 or year > 2100: + raise ValueError(f"Time Machine variant {variant!r} out of range") + return year + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + decade_year = _decade_to_year(variant) + rows = service.get_decade_playlist(decade=decade_year, limit=config.limit) + return coerce_tracks(rows) + + +def variant_resolver(deps: Any) -> List[str]: + """Return the standard decade set. Future enhancement: filter to + decades that actually have data in the discovery pool.""" + return list(_DEFAULT_DECADES) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Time Machine — {variant}', + description='Tracks from a specific decade. One playlist per decade.', + default_config=PlaylistConfig(limit=100, max_per_album=3, max_per_artist=5), + generator=generate, + variant_resolver=variant_resolver, + requires_variant=True, + tags=['time'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/manager.py b/core/personalized/manager.py new file mode 100644 index 00000000..2e936e42 --- /dev/null +++ b/core/personalized/manager.py @@ -0,0 +1,478 @@ +"""Storage layer + lifecycle for personalized playlists. + +The manager is the ONLY place that touches the +``personalized_playlists`` / ``personalized_playlist_tracks`` / +``personalized_track_history`` tables. Generators (in +``core/personalized/generators/``) produce track lists; the manager +persists, refreshes, and serves them. + +Key invariants: + +- ``(profile_id, kind, variant)`` uniquely identifies a playlist. + Variant '' (empty string) means singleton — used for kinds like + ``hidden_gems`` that don't have multiple instances per profile. +- A playlist row is auto-created on first access (``ensure_playlist``) + using the kind's default config. +- Track lists are atomically replaced on refresh — never partial- + mutated. Either the new snapshot lands fully or the old one + remains. +- Refresh failures get logged on the row + (``last_generation_error``) so the UI can surface them without + losing the previous good snapshot. +- Staleness history is append-only and queried by the + ``exclude_recent_days`` config option (handled by individual + generators when they want to honor it). +""" + +from __future__ import annotations + +import json +import time +from dataclasses import asdict +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from utils.logging_config import get_logger + +from core.personalized.specs import PlaylistKindRegistry, get_registry +from core.personalized.types import PlaylistConfig, PlaylistRecord, Track + +logger = get_logger("personalized.manager") + + +class PersonalizedPlaylistManager: + """Owns persistence + refresh lifecycle for personalized playlists.""" + + def __init__(self, database, deps: Any, registry: Optional[PlaylistKindRegistry] = None): + """ + Args: + database: MusicDatabase singleton (exposes ``_get_connection``). + deps: Opaque object passed through to each generator + callable. Whatever a generator needs (the legacy + ``PersonalizedPlaylistsService`` instance, the + ``config_manager``, a metadata client) goes in here. + Manager doesn't inspect it. + registry: optional PlaylistKindRegistry override (for tests). + """ + self.database = database + self.deps = deps + self.registry = registry or get_registry() + + # ─── playlist row lifecycle ────────────────────────────────────── + + def ensure_playlist(self, kind: str, variant: str = '', profile_id: int = 1) -> PlaylistRecord: + """Return the playlist row for ``(profile, kind, variant)``, + creating it from the kind's default config if it doesn't exist.""" + spec = self.registry.get(kind) + if spec is None: + raise ValueError(f"Unknown playlist kind: {kind!r}") + if spec.requires_variant and not variant: + raise ValueError(f"Kind {kind!r} requires a variant") + + existing = self._fetch_playlist_row(kind, variant, profile_id) + if existing is not None: + return existing + + # Insert new row using the kind's defaults. + config = spec.default_config + name = spec.display_name(variant) + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + INSERT INTO personalized_playlists + (profile_id, kind, variant, name, config_json, + track_count, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (profile_id, kind, variant, name, json.dumps(config.to_json_dict())), + ) + conn.commit() + return self._fetch_playlist_row(kind, variant, profile_id) # type: ignore[return-value] + + def get_playlist(self, kind: str, variant: str = '', profile_id: int = 1) -> Optional[PlaylistRecord]: + """Return the playlist row if it exists. Does NOT auto-create.""" + return self._fetch_playlist_row(kind, variant, profile_id) + + def list_playlists(self, profile_id: int = 1) -> List[PlaylistRecord]: + """List every persisted playlist for a profile, newest-first.""" + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT id, profile_id, kind, variant, name, config_json, + track_count, last_generated_at, last_synced_at, + last_generation_source, last_generation_error, + is_stale + FROM personalized_playlists + WHERE profile_id = ? + ORDER BY COALESCE(last_generated_at, created_at) DESC + """, + (profile_id,), + ) + rows = cursor.fetchall() + return [self._row_to_record(r) for r in rows] + + def update_config(self, kind: str, variant: str, profile_id: int, overrides: Dict[str, Any]) -> PlaylistRecord: + """Patch the per-playlist config with the provided overrides.""" + record = self.ensure_playlist(kind, variant, profile_id) + merged = record.config.merged(overrides) + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE personalized_playlists + SET config_json = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (json.dumps(merged.to_json_dict()), record.id), + ) + conn.commit() + return self._fetch_playlist_row(kind, variant, profile_id) # type: ignore[return-value] + + # ─── refresh / generation ───────────────────────────────────────── + + def refresh_playlist( + self, + kind: str, + variant: str = '', + profile_id: int = 1, + config_overrides: Optional[Dict[str, Any]] = None, + ) -> PlaylistRecord: + """Run the kind's generator and persist the result as the + playlist's current snapshot. + + Atomic: track list is replaced in a single transaction. On + generator exception, the previous snapshot is preserved and + the error is recorded on the row. + + Args: + kind: registered kind identifier. + variant: e.g. '1980s' for time machine, '' for singletons. + profile_id: profile to refresh under. + config_overrides: optional per-call config tweaks merged on + top of the stored config (e.g. UI lets the user "preview + with limit=100" without persisting that change). + + Returns: + Updated PlaylistRecord with fresh ``track_count`` / + ``last_generated_at`` (or ``last_generation_error`` on + failure). + """ + spec = self.registry.get(kind) + if spec is None: + raise ValueError(f"Unknown playlist kind: {kind!r}") + record = self.ensure_playlist(kind, variant, profile_id) + + config = record.config + if config_overrides: + config = config.merged(config_overrides) + + try: + tracks = spec.generator(self.deps, variant, config) + except Exception as exc: # noqa: BLE001 — record + re-raise after persisting + logger.exception("Generator failed for kind=%s variant=%s: %s", kind, variant, exc) + self._record_generation_failure(record.id, str(exc)) + return self._fetch_playlist_row(kind, variant, profile_id) # type: ignore[return-value] + + # Quality post-filters — applied uniformly to every kind so + # generators stay focused on selection logic, not staleness + # bookkeeping. Filters are config-driven; defaults preserve + # the pre-overhaul behavior (no filtering). + tracks = self._apply_quality_filters(tracks, kind, profile_id, config) + + return self._persist_snapshot(record.id, kind, profile_id, tracks) + + def _apply_quality_filters( + self, + tracks: List[Track], + kind: str, + profile_id: int, + config: PlaylistConfig, + ) -> List[Track]: + """Apply manager-level quality filters to a generator's output. + + Currently: + - **Staleness window** (`config.exclude_recent_days > 0`): drops + any track whose primary id was served by this `kind` for this + `profile_id` in the last N days. Prevents the same track + from showing up across consecutive refreshes — e.g. a daily + Discovery Shuffle that shouldn't replay yesterday's picks. + Tracks without a primary id pass through unchanged (nothing + to dedupe on). + + Returns a new list (never mutates input). When no filter + applies, returns ``tracks`` unchanged.""" + if config.exclude_recent_days <= 0 or not tracks: + return tracks + + recent_set = set(self.recent_track_ids(profile_id, kind, config.exclude_recent_days)) + if not recent_set: + return tracks + + return [t for t in tracks if not t.primary_id() or t.primary_id() not in recent_set] + + # ─── track read ────────────────────────────────────────────────── + + def get_playlist_tracks(self, playlist_id: int) -> List[Track]: + """Return the persisted track list for a playlist row.""" + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT spotify_track_id, itunes_track_id, deezer_track_id, + track_name, artist_name, album_name, album_cover_url, + duration_ms, popularity, track_data_json + FROM personalized_playlist_tracks + WHERE playlist_id = ? + ORDER BY position ASC + """, + (playlist_id,), + ) + rows = cursor.fetchall() + return [self._row_to_track(r) for r in rows] + + # ─── snapshot freshness vs source data ─────────────────────────── + + def mark_kinds_stale(self, kinds: List[str], profile_id: Optional[int] = None) -> int: + """Flag every playlist row matching one of ``kinds`` as stale. + + Called by upstream data refreshers (watchlist scan finishing + / Spotify enrichment worker re-pulling Release Radar / etc) + so pipelines auto-regenerate snapshots before the next sync + instead of pushing stale data to the media server. + + Returns the number of rows touched. When ``profile_id`` is + None, flags rows across every profile. + """ + if not kinds: + return 0 + placeholders = ','.join('?' * len(kinds)) + sql = f"UPDATE personalized_playlists SET is_stale = 1, updated_at = CURRENT_TIMESTAMP WHERE kind IN ({placeholders})" + params: List[Any] = list(kinds) + if profile_id is not None: + sql += " AND profile_id = ?" + params.append(profile_id) + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(sql, params) + count = cursor.rowcount + conn.commit() + return count + + # ─── staleness history ─────────────────────────────────────────── + + def recent_track_ids(self, profile_id: int, kind: str, days: int) -> List[str]: + """Return track IDs served by ``kind`` for ``profile_id`` in + the last ``days`` days. Generators query this when honoring + the ``exclude_recent_days`` config knob.""" + if days <= 0: + return [] + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT DISTINCT track_id + FROM personalized_track_history + WHERE profile_id = ? + AND kind = ? + AND served_at >= datetime('now', ?) + """, + (profile_id, kind, f'-{int(days)} days'), + ) + return [r[0] for r in cursor.fetchall() if r[0]] + + # ─── internal helpers ──────────────────────────────────────────── + + def _persist_snapshot(self, playlist_id: int, kind: str, profile_id: int, tracks: List[Track]) -> PlaylistRecord: + """Atomic replace of a playlist's track list + history append.""" + now = datetime.now(timezone.utc).isoformat(timespec='seconds') + primary_source = next( + (t.source for t in tracks if t.source), None, + ) + with self.database._get_connection() as conn: + cursor = conn.cursor() + try: + cursor.execute("BEGIN") + cursor.execute( + "DELETE FROM personalized_playlist_tracks WHERE playlist_id = ?", + (playlist_id,), + ) + for position, track in enumerate(tracks): + td = track.track_data_json + if td is not None and not isinstance(td, str): + td = json.dumps(td) + cursor.execute( + """ + INSERT INTO personalized_playlist_tracks + (playlist_id, position, + spotify_track_id, itunes_track_id, deezer_track_id, + track_name, artist_name, album_name, album_cover_url, + duration_ms, popularity, track_data_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + playlist_id, position, + track.spotify_track_id, track.itunes_track_id, track.deezer_track_id, + track.track_name, track.artist_name, track.album_name, track.album_cover_url, + int(track.duration_ms or 0), int(track.popularity or 0), td, + ), + ) + cursor.execute( + """ + UPDATE personalized_playlists + SET track_count = ?, last_generated_at = CURRENT_TIMESTAMP, + last_generation_source = ?, last_generation_error = NULL, + is_stale = 0, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (len(tracks), primary_source, playlist_id), + ) + # History append — only for tracks with a primary ID, + # used by exclude_recent_days filter on next refresh. + for track in tracks: + tid = track.primary_id() + if not tid: + continue + cursor.execute( + """ + INSERT INTO personalized_track_history + (profile_id, kind, track_id, served_at) + VALUES (?, ?, ?, CURRENT_TIMESTAMP) + """, + (profile_id, kind, tid), + ) + conn.commit() + except Exception: + conn.rollback() + raise + + # Re-read the row so the returned record carries the fresh + # last_generated_at timestamp. + record = self._fetch_playlist_row_by_id(playlist_id) + if record is None: + raise RuntimeError(f"playlist row {playlist_id} disappeared mid-refresh") + return record + + def _record_generation_failure(self, playlist_id: int, error_message: str) -> None: + """Stamp the error on the row WITHOUT touching tracks.""" + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE personalized_playlists + SET last_generation_error = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (error_message[:500], playlist_id), + ) + conn.commit() + + def _fetch_playlist_row(self, kind: str, variant: str, profile_id: int) -> Optional[PlaylistRecord]: + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT id, profile_id, kind, variant, name, config_json, + track_count, last_generated_at, last_synced_at, + last_generation_source, last_generation_error, + is_stale + FROM personalized_playlists + WHERE profile_id = ? AND kind = ? AND variant = ? + """, + (profile_id, kind, variant), + ) + row = cursor.fetchone() + return self._row_to_record(row) if row else None + + def _fetch_playlist_row_by_id(self, playlist_id: int) -> Optional[PlaylistRecord]: + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT id, profile_id, kind, variant, name, config_json, + track_count, last_generated_at, last_synced_at, + last_generation_source, last_generation_error, + is_stale + FROM personalized_playlists + WHERE id = ? + """, + (playlist_id,), + ) + row = cursor.fetchone() + return self._row_to_record(row) if row else None + + @staticmethod + def _row_to_record(row: Any) -> PlaylistRecord: + # Tolerate sqlite3.Row + plain tuples. + if hasattr(row, 'keys'): + row = dict(row) + return PlaylistRecord( + id=row['id'], profile_id=row['profile_id'], + kind=row['kind'], variant=row['variant'] or '', + name=row['name'], + config=PlaylistConfig.from_json_dict(_safe_json_loads(row['config_json'])), + track_count=row['track_count'] or 0, + last_generated_at=row.get('last_generated_at'), + last_synced_at=row.get('last_synced_at'), + last_generation_source=row.get('last_generation_source'), + last_generation_error=row.get('last_generation_error'), + is_stale=bool(row.get('is_stale') or 0), + ) + # Tuple form: positional access matches SELECT order above. + return PlaylistRecord( + id=row[0], profile_id=row[1], + kind=row[2], variant=row[3] or '', + name=row[4], + config=PlaylistConfig.from_json_dict(_safe_json_loads(row[5])), + track_count=row[6] or 0, + last_generated_at=row[7], + last_synced_at=row[8], + last_generation_source=row[9], + last_generation_error=row[10], + is_stale=bool(row[11] or 0) if len(row) > 11 else False, + ) + + @staticmethod + def _row_to_track(row: Any) -> Track: + if hasattr(row, 'keys'): + row = dict(row) + return Track( + spotify_track_id=row.get('spotify_track_id'), + itunes_track_id=row.get('itunes_track_id'), + deezer_track_id=row.get('deezer_track_id'), + track_name=row.get('track_name', ''), + artist_name=row.get('artist_name', ''), + album_name=row.get('album_name') or '', + album_cover_url=row.get('album_cover_url'), + duration_ms=int(row.get('duration_ms') or 0), + popularity=int(row.get('popularity') or 0), + track_data_json=_safe_json_loads(row.get('track_data_json')), + ) + return Track( + spotify_track_id=row[0], itunes_track_id=row[1], deezer_track_id=row[2], + track_name=row[3], artist_name=row[4], album_name=row[5] or '', + album_cover_url=row[6], duration_ms=int(row[7] or 0), + popularity=int(row[8] or 0), + track_data_json=_safe_json_loads(row[9]), + ) + + +def _safe_json_loads(value: Any) -> Any: + """Tolerant JSON parse — returns None / dict / passes through + non-string values. Avoids exceptions on bad rows so the manager + never breaks on a single corrupt record.""" + if value is None: + return None + if not isinstance(value, str): + return value + if not value.strip(): + return None + try: + return json.loads(value) + except (ValueError, TypeError): + return None + + +__all__ = ['PersonalizedPlaylistManager'] diff --git a/core/personalized/specs.py b/core/personalized/specs.py new file mode 100644 index 00000000..d1367d9c --- /dev/null +++ b/core/personalized/specs.py @@ -0,0 +1,121 @@ +"""Per-kind specifications for the personalized-playlist subsystem. + +A ``PlaylistKindSpec`` declares everything the manager needs to know +about one playlist type: + +- The kind identifier (stable string used in URLs / configs / DB). +- Human-readable display name template (with optional ``{variant}`` + substitution). +- Whether the kind supports / requires variants and what valid + variants look like. +- The default user-tweakable config for this kind. +- A generator callable that produces a fresh track list given + ``(deps, variant, config)``. + +Generators live in ``core/personalized/generators/`` (added in +later commits as each kind is migrated). For commit 1 the registry +ships empty — schema + manager land first; generators arrive +incrementally with their per-kind tests. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional + +from core.personalized.types import PlaylistConfig, Track + + +# Generator callable signature. +# Args: +# deps: opaque object the generator may need (DB, service handle, +# config_manager). Each generator declares what it pulls. +# variant: e.g. '1980s' for time machine, 'halloween' for seasonal. +# Empty string '' for singleton kinds. +# config: PlaylistConfig with the user's per-playlist overrides +# merged onto the kind's defaults. +# Returns: +# List[Track] — the fresh snapshot to persist as the playlist's +# current track list. +GeneratorFn = Callable[[Any, str, PlaylistConfig], List[Track]] + + +# Variant resolver: returns the list of currently-valid variant +# identifiers for a kind that supports multiple instances. Used by +# the manager when auto-creating playlist rows for newly available +# variants (e.g. a new decade, a new season). Singletons return ['']. +VariantResolver = Callable[[Any], List[str]] + + +@dataclass +class PlaylistKindSpec: + """Declaration of one playlist kind. + + See module docstring for the contract. + """ + + kind: str + name_template: str # e.g. 'Time Machine — {variant}', 'Hidden Gems' + description: str + default_config: PlaylistConfig + generator: GeneratorFn + variant_resolver: Optional[VariantResolver] = None + requires_variant: bool = False + # Tags for UI grouping ('curated' / 'discovery' / 'time' / 'genre'). + tags: List[str] = field(default_factory=list) + + def display_name(self, variant: str) -> str: + """Render the human-readable playlist name for a given variant.""" + if not variant: + return self.name_template.replace('{variant}', '').strip(' —-') + return self.name_template.format(variant=variant) + + +class PlaylistKindRegistry: + """Module-level registry of every kind the manager knows about. + + Populated at import time as each generator module is loaded. The + manager queries the registry at runtime to dispatch refresh + requests, list available kinds for the UI, and resolve variants. + """ + + def __init__(self) -> None: + self._kinds: Dict[str, PlaylistKindSpec] = {} + + def register(self, spec: PlaylistKindSpec) -> None: + if spec.kind in self._kinds: + raise ValueError(f"Kind {spec.kind!r} already registered") + self._kinds[spec.kind] = spec + + def get(self, kind: str) -> Optional[PlaylistKindSpec]: + return self._kinds.get(kind) + + def all(self) -> List[PlaylistKindSpec]: + return list(self._kinds.values()) + + def kinds(self) -> List[str]: + return list(self._kinds.keys()) + + def reset_for_tests(self) -> None: + """Drop every registration. Tests only — production runs + register at module import and never reset.""" + self._kinds.clear() + + +# Module-level singleton. Generators register against this on import. +_registry = PlaylistKindRegistry() + + +def get_registry() -> PlaylistKindRegistry: + """Public accessor for the module-level registry. Tests can reset + via ``get_registry().reset_for_tests()``.""" + return _registry + + +__all__ = [ + 'PlaylistKindSpec', + 'PlaylistKindRegistry', + 'GeneratorFn', + 'VariantResolver', + 'get_registry', +] diff --git a/core/personalized/types.py b/core/personalized/types.py new file mode 100644 index 00000000..ef0dccae --- /dev/null +++ b/core/personalized/types.py @@ -0,0 +1,181 @@ +"""Shared dataclasses for the personalized-playlist subsystem. + +These are pure data containers — no business logic, no IO. The +manager + specs + generators all speak in these types so the seam +between them stays mechanical. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class Track: + """A track in a personalized playlist. + + Mirrors the shape returned by + ``PersonalizedPlaylistsService._build_track_dict`` so the legacy + generators can be wrapped without translating fields. Always at + least one of the source IDs is populated; ``track_data_json`` is + the full enriched track object when available (used by sync / + download paths that need richer metadata than just the ID).""" + + track_name: str + artist_name: str + album_name: str = '' + spotify_track_id: Optional[str] = None + itunes_track_id: Optional[str] = None + deezer_track_id: Optional[str] = None + album_cover_url: Optional[str] = None + duration_ms: int = 0 + popularity: int = 0 + track_data_json: Optional[Any] = None # dict OR JSON string OR None + source: Optional[str] = None + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> 'Track': + """Coerce a legacy generator's output dict into a Track. + + Tolerates the ``_artist_genres_raw`` / ``_release_date`` extra- + column passthroughs by ignoring them — this dataclass only + carries the storage-layer fields.""" + return cls( + track_name=d.get('track_name', 'Unknown'), + artist_name=d.get('artist_name', 'Unknown'), + album_name=d.get('album_name', '') or '', + spotify_track_id=d.get('spotify_track_id'), + itunes_track_id=d.get('itunes_track_id'), + deezer_track_id=d.get('deezer_track_id'), + album_cover_url=d.get('album_cover_url'), + duration_ms=int(d.get('duration_ms') or 0), + popularity=int(d.get('popularity') or 0), + track_data_json=d.get('track_data_json'), + source=d.get('source'), + ) + + def primary_id(self) -> Optional[str]: + """Return the first non-empty source ID. Used as the staleness- + history key + the dedupe key when persisting tracks.""" + return ( + self.spotify_track_id + or self.itunes_track_id + or self.deezer_track_id + or None + ) + + +@dataclass +class PlaylistConfig: + """User-tweakable knobs per playlist instance. + + Stored as JSON in `personalized_playlists.config_json`. Defaults + come from the kind's spec; user overrides override per-playlist. + + Fields: + - limit: target number of tracks + - max_per_album / max_per_artist: diversity caps + - popularity_min / popularity_max: filter bounds (None = ignore) + - exclude_recent_days: avoid tracks served by this kind in the + last N days (0 = no exclusion) + - recency_days: only include tracks released in the last N days + (None = all-time) + - seed: optional deterministic seed for randomization (None = + use system random; same seed + same pool = same output) + - extra: free-form per-kind extension dict (e.g. seasonal mix + stores ``selected_seasons``, time machine stores + ``selected_decades``, genre stores ``selected_genres``). + """ + + limit: int = 50 + max_per_album: int = 2 + max_per_artist: int = 3 + popularity_min: Optional[int] = None + popularity_max: Optional[int] = None + exclude_recent_days: int = 0 + recency_days: Optional[int] = None + seed: Optional[int] = None + extra: Dict[str, Any] = field(default_factory=dict) + + def to_json_dict(self) -> Dict[str, Any]: + """Serialise to a JSON-safe dict for storage.""" + return { + 'limit': self.limit, + 'max_per_album': self.max_per_album, + 'max_per_artist': self.max_per_artist, + 'popularity_min': self.popularity_min, + 'popularity_max': self.popularity_max, + 'exclude_recent_days': self.exclude_recent_days, + 'recency_days': self.recency_days, + 'seed': self.seed, + 'extra': dict(self.extra), + } + + @classmethod + def from_json_dict(cls, d: Optional[Dict[str, Any]]) -> 'PlaylistConfig': + """Reconstruct from a stored JSON dict. Missing fields fall + back to defaults so old rows + new code stay compatible.""" + if not isinstance(d, dict): + return cls() + return cls( + limit=int(d.get('limit', 50)), + max_per_album=int(d.get('max_per_album', 2)), + max_per_artist=int(d.get('max_per_artist', 3)), + popularity_min=d.get('popularity_min'), + popularity_max=d.get('popularity_max'), + exclude_recent_days=int(d.get('exclude_recent_days', 0)), + recency_days=d.get('recency_days'), + seed=d.get('seed'), + extra=dict(d.get('extra') or {}), + ) + + def merged(self, overrides: Dict[str, Any]) -> 'PlaylistConfig': + """Return a new PlaylistConfig with `overrides` merged in. + + Used when a user PATCHes their per-playlist config — apply + only the fields they sent, leave the rest at their stored + values.""" + base = self.to_json_dict() + for key, value in (overrides or {}).items(): + if key == 'extra' and isinstance(value, dict): + base['extra'] = {**base.get('extra', {}), **value} + elif key in base: + base[key] = value + return PlaylistConfig.from_json_dict(base) + + +@dataclass +class PlaylistRecord: + """One row of `personalized_playlists` plus its track count. + + The live track list is fetched separately via + ``PersonalizedPlaylistManager.get_playlist_tracks(playlist_id)`` + so list / detail responses can stay cheap when the caller only + needs metadata. + + ``is_stale`` flips to True when the underlying source data + changes (e.g. watchlist scan updates the discovery pool) and is + cleared on the next successful refresh. Pipelines auto-refresh + stale snapshots before syncing so the server playlist always + reflects the latest source data.""" + + id: int + profile_id: int + kind: str + variant: str + name: str + config: PlaylistConfig + track_count: int + last_generated_at: Optional[str] + last_synced_at: Optional[str] + last_generation_source: Optional[str] + last_generation_error: Optional[str] + is_stale: bool = False + + +__all__ = [ + 'Track', + 'PlaylistConfig', + 'PlaylistRecord', +] diff --git a/core/personalized_playlists.py b/core/personalized_playlists.py index 64882b97..ec2505dd 100644 --- a/core/personalized_playlists.py +++ b/core/personalized_playlists.py @@ -779,19 +779,21 @@ class PersonalizedPlaylistsService: } def _get_library_tracks_by_category(self, category: str, limit: int) -> List[Dict]: - """ - Get tracks from library matching genre or artist + """Library tracks intentionally excluded from daily mixes. - NOTE: This requires library tracks to have Spotify metadata which may not be available. - Returns empty list if schema incompatible. - """ - try: - logger.warning("Library tracks by category requires Spotify-linked library - returning empty") - return [] + The legacy ambition was 50% library + 50% discovery, but the + ``tracks`` table doesn't carry source IDs (no + ``spotify_track_id`` / ``itunes_track_id`` / ``deezer_track_id`` + column) — so library rows can't flow through the same + sync / wishlist pipeline that discovery tracks do. Returning + them here would produce un-syncable, un-downloadable phantom + entries. - except Exception as e: - logger.error(f"Error getting library tracks by category: {e}") - return [] + Returns ``[]`` so callers compose with ``_get_discovery_tracks_by_category`` + for a discovery-only mix. A future PR can backfill source IDs + into library rows and lift this restriction. + """ + return [] def _get_discovery_tracks_by_category(self, category: str, limit: int) -> List[Dict]: """Get tracks from discovery pool matching genre or artist""" @@ -900,7 +902,9 @@ class PersonalizedPlaylistsService: with self.database._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" - SELECT similar_artist_spotify_id, similar_artist_name + SELECT similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_deezer_id, similar_artist_musicbrainz_id, + similar_artist_name FROM similar_artists WHERE source_artist_id = ? ORDER BY similarity_rank ASC @@ -909,9 +913,16 @@ class PersonalizedPlaylistsService: db_results = cursor.fetchall() if db_results: + source_id_col = { + 'spotify': 'similar_artist_spotify_id', + 'itunes': 'similar_artist_itunes_id', + 'deezer': 'similar_artist_deezer_id', + 'musicbrainz': 'similar_artist_musicbrainz_id', + }.get(active_source, 'similar_artist_itunes_id') for row in db_results: - artist_id = row['similar_artist_spotify_id'] - artist_name = row['similar_artist_name'] + r = dict(row) + artist_id = r.get(source_id_col) or r.get('similar_artist_spotify_id') or r.get('similar_artist_itunes_id') + artist_name = r['similar_artist_name'] if artist_id and artist_id not in seen_artist_ids: all_similar_artists.append({'id': artist_id, 'name': artist_name}) seen_artist_ids.add(artist_id) diff --git a/core/prowlarr_client.py b/core/prowlarr_client.py new file mode 100644 index 00000000..4290f336 --- /dev/null +++ b/core/prowlarr_client.py @@ -0,0 +1,241 @@ +"""Prowlarr client — indexer aggregator. + +Prowlarr is the indexer manager component of the *arr stack. It exposes +configured Usenet / torrent indexers behind a single Newznab-style API +so downstream apps (Lidarr, Sonarr, Radarr, SoulSync) don't have to +implement an indexer integration per provider. + +This client is NOT a download source plugin. It does not implement +``DownloadSourcePlugin`` — Prowlarr only *searches*. The torrent / +usenet download plugins (built in subsequent commits) own the +add-to-client / poll-status / extract flow and call this client for +the search step. + +Surface: +- ``is_configured()`` — URL + API key present. +- ``check_connection()`` — hits ``/api/v1/system/status``. +- ``get_indexers()`` — list of configured indexers (id, name, protocol, + capabilities). +- ``search(query, categories, indexer_ids)`` — Newznab search across + selected indexers. Music categories default to the full audio tree. + +Auth: ``X-Api-Key`` header. Found in Prowlarr → Settings → General → +Security → API Key. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence + +import requests as http_requests + +from config.settings import config_manager +from utils.logging_config import get_logger + +logger = get_logger("prowlarr_client") + + +# Newznab Music category tree. Prowlarr / Jackett / Newznab indexers +# all agree on these numeric IDs. 3000 is the parent — most indexers +# tag releases against the parent OR a leaf; searching the parent +# pulls everything. +MUSIC_CATEGORY_ALL = 3000 +MUSIC_CATEGORY_MP3 = 3010 +MUSIC_CATEGORY_VIDEO = 3020 +MUSIC_CATEGORY_AUDIOBOOK = 3030 +MUSIC_CATEGORY_LOSSLESS = 3040 +MUSIC_CATEGORY_OTHER = 3050 +MUSIC_CATEGORY_FOREIGN = 3060 + +DEFAULT_MUSIC_CATEGORIES: tuple = ( + MUSIC_CATEGORY_ALL, + MUSIC_CATEGORY_MP3, + MUSIC_CATEGORY_LOSSLESS, + MUSIC_CATEGORY_OTHER, +) + + +@dataclass +class ProwlarrIndexer: + """One configured indexer exposed by Prowlarr.""" + + id: int + name: str + protocol: str # "torrent" | "usenet" + enable: bool + privacy: str # "public" | "private" | "semiPrivate" + categories: List[int] = field(default_factory=list) + capabilities: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ProwlarrSearchResult: + """One release returned by a Prowlarr search. + + ``download_url`` is the link the torrent / usenet client gets fed. + For torrent indexers it may be either a ``.torrent`` HTTP URL or + a magnet URI (sometimes both — ``magnet_uri`` is set when the + indexer exposes the magnet separately). + """ + + guid: str + title: str + indexer_id: int + indexer_name: str + protocol: str # "torrent" | "usenet" + download_url: Optional[str] = None + magnet_uri: Optional[str] = None + info_url: Optional[str] = None + size: int = 0 # bytes + seeders: Optional[int] = None + leechers: Optional[int] = None + grabs: Optional[int] = None + publish_date: Optional[str] = None + categories: List[int] = field(default_factory=list) + raw: Dict[str, Any] = field(default_factory=dict) + + +class ProwlarrClient: + """Thin sync-backed async wrapper around the Prowlarr v1 API.""" + + DEFAULT_TIMEOUT = 15 + + def __init__(self) -> None: + self._load_config() + + def _load_config(self) -> None: + self._url = (config_manager.get('prowlarr.url', '') or '').rstrip('/') + self._api_key = config_manager.get('prowlarr.api_key', '') or '' + + def reload_settings(self) -> None: + self._load_config() + logger.info("Prowlarr settings reloaded") + + def is_configured(self) -> bool: + return bool(self._url and self._api_key) + + async def check_connection(self) -> bool: + if not self.is_configured(): + return False + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._check_connection_sync) + + def _check_connection_sync(self) -> bool: + data = self._api_get('system/status') + return bool(data and 'version' in data) + + async def get_indexers(self) -> List[ProwlarrIndexer]: + if not self.is_configured(): + return [] + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_indexers_sync) + + def _get_indexers_sync(self) -> List[ProwlarrIndexer]: + data = self._api_get('indexer') + if not isinstance(data, list): + return [] + return [self._parse_indexer(entry) for entry in data if isinstance(entry, dict)] + + async def search( + self, + query: str, + categories: Sequence[int] = DEFAULT_MUSIC_CATEGORIES, + indexer_ids: Optional[Sequence[int]] = None, + limit: int = 100, + ) -> List[ProwlarrSearchResult]: + """Run a Newznab search across the selected indexers. + + ``indexer_ids`` is the list of Prowlarr internal indexer IDs to + query. ``None`` means all enabled indexers. + """ + if not self.is_configured() or not query.strip(): + return [] + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._search_sync, query, list(categories), list(indexer_ids or []), limit + ) + + def _search_sync( + self, + query: str, + categories: List[int], + indexer_ids: List[int], + limit: int, + ) -> List[ProwlarrSearchResult]: + # Prowlarr's search endpoint accepts repeated params: ``categories=3000&categories=3010``. + # ``requests`` serializes lists in that exact form when passed as tuples of pairs. + params: List[tuple] = [('query', query), ('type', 'search'), ('limit', limit)] + for cat in categories: + params.append(('categories', cat)) + for indexer_id in indexer_ids: + params.append(('indexerIds', indexer_id)) + + data = self._api_get('search', params=params) + if not isinstance(data, list): + return [] + return [self._parse_result(entry) for entry in data if isinstance(entry, dict)] + + def _parse_indexer(self, entry: Dict[str, Any]) -> ProwlarrIndexer: + return ProwlarrIndexer( + id=int(entry.get('id') or 0), + name=entry.get('name') or '', + protocol=entry.get('protocol') or '', + enable=bool(entry.get('enable', True)), + privacy=entry.get('privacy') or '', + categories=[int(c.get('id') or 0) for c in entry.get('capabilities', {}).get('categories', []) if isinstance(c, dict)], + capabilities=entry.get('capabilities', {}) or {}, + ) + + def _parse_result(self, entry: Dict[str, Any]) -> ProwlarrSearchResult: + cats = entry.get('categories') or [] + category_ids: List[int] = [] + for cat in cats: + if isinstance(cat, dict) and cat.get('id') is not None: + try: + category_ids.append(int(cat['id'])) + except (TypeError, ValueError): + continue + elif isinstance(cat, int): + category_ids.append(cat) + + return ProwlarrSearchResult( + guid=str(entry.get('guid') or entry.get('infoUrl') or entry.get('downloadUrl') or ''), + title=entry.get('title') or '', + indexer_id=int(entry.get('indexerId') or 0), + indexer_name=entry.get('indexer') or '', + protocol=entry.get('protocol') or '', + download_url=entry.get('downloadUrl') or None, + magnet_uri=entry.get('magnetUrl') or None, + info_url=entry.get('infoUrl') or None, + size=int(entry.get('size') or 0), + seeders=entry.get('seeders'), + leechers=entry.get('leechers'), + grabs=entry.get('grabs'), + publish_date=entry.get('publishDate'), + categories=category_ids, + raw=entry, + ) + + def _api_get(self, path: str, params=None) -> Optional[Any]: + if not self.is_configured(): + return None + url = f"{self._url}/api/v1/{path.lstrip('/')}" + try: + resp = http_requests.get( + url, + headers={'X-Api-Key': self._api_key, 'Accept': 'application/json'}, + params=params, + timeout=self.DEFAULT_TIMEOUT, + ) + if not resp.ok: + logger.warning("Prowlarr %s returned HTTP %s", path, resp.status_code) + return None + return resp.json() + except http_requests.exceptions.RequestException as e: + logger.error("Prowlarr request to %s failed: %s", path, e) + return None + except ValueError as e: + logger.error("Prowlarr response to %s was not JSON: %s", path, e) + return None diff --git a/core/qobuz_client.py b/core/qobuz_client.py index 1cb58a66..cf4fbf9b 100644 --- a/core/qobuz_client.py +++ b/core/qobuz_client.py @@ -711,6 +711,249 @@ class QobuzClient(DownloadSourcePlugin): logger.error(f"Error getting Qobuz track {track_id}: {e}") return None + # ===================== Playlists & Favorites ===================== + # + # Qobuz playlist sync surface — mirrors the Tidal client contract + # (see core/tidal_client.py:629 + :1227) so the Sync page's + # per-service handlers can render Qobuz playlists in the same + # discovery / mirror flow. Returns normalized dicts rather than + # dataclasses to match the rest of this client's idiom. + # + # Favorite Tracks ride on the same `get_playlist()` entry point via + # the virtual ID below — same pattern as Tidal's COLLECTION_PLAYLIST_ID + # so sync-services.js can treat favorites as just another playlist + # card without per-service special-casing. + QOBUZ_FAVORITES_ID = "qobuz-favorites" + QOBUZ_FAVORITES_NAME = "Favorite Tracks" + QOBUZ_FAVORITES_DESCRIPTION = "Your favorited tracks on Qobuz" + + # Page size for paginated playlist + favorite listings. Qobuz caps at + # 500 per page; 100 is a safe middle ground for responsiveness. + _PLAYLIST_PAGE_SIZE = 100 + + def _normalize_qobuz_playlist(self, p: Dict) -> Dict: + """Project a Qobuz playlist dict into the shape the Sync page expects.""" + image = p.get('images', []) or [] + image_url = image[0] if image else (p.get('image_rectangle', [None])[0] if p.get('image_rectangle') else None) + if not image_url: + image_url = p.get('image', '') or '' + return { + 'id': str(p.get('id', '')), + 'name': p.get('name', 'Unknown Playlist'), + 'description': p.get('description', '') or '', + 'public': bool(p.get('is_public', False)), + 'track_count': int(p.get('tracks_count', 0) or 0), + 'image_url': image_url, + 'external_urls': {'qobuz': f"https://play.qobuz.com/playlist/{p.get('id', '')}"} if p.get('id') else {}, + } + + def _normalize_qobuz_track(self, t: Dict) -> Dict: + """Project a Qobuz track dict into the shape the Sync page expects.""" + performer = t.get('performer') or {} + album = t.get('album') or {} + album_artist = album.get('artist') or {} + + # Artist names — Qobuz can stash the artist on performer, album.artist, + # or composer depending on the track. Prefer performer, fall back to + # album artist, then composer, then "Unknown Artist". + artist_name = ( + performer.get('name') + or album_artist.get('name') + or (t.get('composer') or {}).get('name') + or 'Unknown Artist' + ) + + album_image = album.get('image') or {} + image_url = album_image.get('large') or album_image.get('small') or album_image.get('thumbnail') or '' + + return { + 'id': str(t.get('id', '')), + 'name': t.get('title', '') or '', + 'artists': [artist_name], + 'album': album.get('title', '') or '', + 'duration_ms': int(t.get('duration', 0) or 0) * 1000, + 'image_url': image_url, + 'external_urls': {'qobuz': f"https://play.qobuz.com/track/{t.get('id', '')}"} if t.get('id') else {}, + 'explicit': bool(t.get('parental_warning', False)), + } + + def get_user_playlists(self) -> List[Dict[str, Any]]: + """Fetch the authenticated user's Qobuz playlists. + + Returns metadata only (no tracks) — track lists are fetched + on-demand via `get_playlist()` when the user selects one. + Matches the Tidal `get_user_playlists_metadata_only` contract + so the Sync page renderer can treat both services uniformly. + """ + if not self.is_authenticated(): + logger.warning("Qobuz not authenticated — cannot list playlists") + return [] + + playlists: List[Dict[str, Any]] = [] + offset = 0 + while True: + data = self._api_request('playlist/getUserPlaylists', { + 'limit': self._PLAYLIST_PAGE_SIZE, + 'offset': offset, + }) + if not data: + break + + container = data.get('playlists') or {} + items = container.get('items', []) or [] + if not items: + break + + for raw in items: + try: + playlists.append(self._normalize_qobuz_playlist(raw)) + except Exception as exc: + logger.debug(f"Skipping malformed Qobuz playlist entry: {exc}") + + total = int(container.get('total', len(playlists)) or len(playlists)) + offset += len(items) + if offset >= total or len(items) < self._PLAYLIST_PAGE_SIZE: + break + + logger.info(f"Retrieved {len(playlists)} Qobuz user playlists") + return playlists + + def get_playlist(self, playlist_id: str) -> Optional[Dict[str, Any]]: + """Fetch a Qobuz playlist with its full tracklist. + + Recognizes the virtual ``qobuz-favorites`` ID and dispatches to + ``get_user_favorite_tracks`` so the Sync page can treat + favorites as just another playlist card (same pattern as + Tidal's ``tidal-favorites``). + """ + if not self.is_authenticated(): + logger.warning("Qobuz not authenticated — cannot fetch playlist") + return None + + if str(playlist_id) == self.QOBUZ_FAVORITES_ID: + tracks = self.get_user_favorite_tracks() + return { + 'id': self.QOBUZ_FAVORITES_ID, + 'name': self.QOBUZ_FAVORITES_NAME, + 'description': self.QOBUZ_FAVORITES_DESCRIPTION, + 'public': False, + 'track_count': len(tracks), + 'image_url': '', + 'external_urls': {}, + 'tracks': tracks, + } + + # Paginate tracks ourselves — Qobuz's playlist/get only returns + # the first ~50 tracks even with limit=500 on some accounts. + tracks: List[Dict[str, Any]] = [] + offset = 0 + playlist_meta: Optional[Dict] = None + while True: + data = self._api_request('playlist/get', { + 'playlist_id': playlist_id, + 'extra': 'tracks', + 'limit': self._PLAYLIST_PAGE_SIZE, + 'offset': offset, + }) + if not data: + break + + if playlist_meta is None: + playlist_meta = data + + track_container = data.get('tracks') or {} + items = track_container.get('items', []) or [] + if not items: + break + + for raw in items: + try: + tracks.append(self._normalize_qobuz_track(raw)) + except Exception as exc: + logger.debug(f"Skipping malformed Qobuz playlist track: {exc}") + + total = int(track_container.get('total', len(tracks)) or len(tracks)) + offset += len(items) + if offset >= total or len(items) < self._PLAYLIST_PAGE_SIZE: + break + + if playlist_meta is None: + logger.warning(f"Qobuz playlist {playlist_id} not found") + return None + + normalized = self._normalize_qobuz_playlist(playlist_meta) + normalized['tracks'] = tracks + normalized['track_count'] = len(tracks) + logger.info(f"Retrieved Qobuz playlist '{normalized['name']}' with {len(tracks)} tracks") + return normalized + + def get_user_favorite_tracks(self, limit: Optional[int] = None) -> List[Dict[str, Any]]: + """Fetch the authenticated user's favorited tracks. + + Mirrors ``TidalClient.get_collection_tracks`` — the Sync page's + Favorite Tracks card pulls from here on click. By default this + fetches the full favorites collection so the card count and the + discovered track list cannot silently diverge. Pass ``limit`` for + explicit capped callers. + """ + if not self.is_authenticated(): + logger.warning("Qobuz not authenticated — cannot list favorite tracks") + return [] + + tracks: List[Dict[str, Any]] = [] + offset = 0 + while True: + page_size = self._PLAYLIST_PAGE_SIZE if limit is None else min(self._PLAYLIST_PAGE_SIZE, limit - len(tracks)) + if page_size <= 0: + break + + data = self._api_request('favorite/getUserFavorites', { + 'type': 'tracks', + 'limit': page_size, + 'offset': offset, + }) + if not data: + break + + container = data.get('tracks') or {} + items = container.get('items', []) or [] + if not items: + break + + for raw in items: + try: + tracks.append(self._normalize_qobuz_track(raw)) + except Exception as exc: + logger.debug(f"Skipping malformed Qobuz favorite track: {exc}") + + total = int(container.get('total', len(tracks)) or len(tracks)) + offset += len(items) + if offset >= total or len(items) < page_size: + break + if limit is not None and len(tracks) >= limit: + break + + logger.info(f"Retrieved {len(tracks)} Qobuz favorite tracks") + return tracks + + def get_user_favorite_tracks_count(self) -> int: + """Cheap track-count lookup for the Favorite Tracks card metadata. + + Mirrors ``TidalClient.get_collection_tracks_count`` — avoids + fetching the full list just to populate the card's track-count + chip on the Sync page. + """ + if not self.is_authenticated(): + return 0 + data = self._api_request('favorite/getUserFavorites', { + 'type': 'tracks', + 'limit': 1, + 'offset': 0, + }) + if not data: + return 0 + return int((data.get('tracks') or {}).get('total', 0) or 0) + async def search(self, query: str, timeout: int = None, progress_callback=None) -> Tuple[List[TrackResult], List[AlbumResult]]: """ Search Qobuz for tracks (async, Soulseek-compatible interface). diff --git a/core/reorganize_queue.py b/core/reorganize_queue.py index 6b50afb0..a86a638c 100644 --- a/core/reorganize_queue.py +++ b/core/reorganize_queue.py @@ -66,6 +66,10 @@ class QueueItem: artist_name: str # captured at enqueue time for UI display source: Optional[str] # the user's per-modal pick (None = auto) enqueued_at: float + # 'api' (default) = query metadata source per album_data IDs. + # 'tags' = read each file's embedded tags as the source + # of truth (issue #592). Zero API calls. + metadata_source: str = 'api' status: str = 'queued' # queued | running | done | failed | cancelled started_at: Optional[float] = None finished_at: Optional[float] = None @@ -91,6 +95,7 @@ class QueueItem: 'artist_id': self.artist_id, 'artist_name': self.artist_name, 'source': self.source, + 'metadata_source': self.metadata_source, 'enqueued_at': self.enqueued_at, 'started_at': self.started_at, 'finished_at': self.finished_at, @@ -155,6 +160,7 @@ class ReorganizeQueue: artist_id: Optional[str], artist_name: str, source: Optional[str] = None, + metadata_source: str = 'api', ) -> dict: """Add an album to the queue. Returns a result dict: @@ -183,6 +189,7 @@ class ReorganizeQueue: artist_name=artist_name, source=source, enqueued_at=time.time(), + metadata_source=metadata_source or 'api', ) self._items.append(item) position = sum(1 for i in self._items if i.status == 'queued') @@ -190,7 +197,8 @@ class ReorganizeQueue: self._cond.notify_all() logger.info( f"[Queue] Enqueued '{album_title}' (album_id={album_id}, " - f"queue_id={item.queue_id}, position={position}, source={source or 'auto'})" + f"queue_id={item.queue_id}, position={position}, " + f"source={source or 'auto'}, metadata={item.metadata_source})" ) return { 'queued': True, @@ -240,6 +248,7 @@ class ReorganizeQueue: artist_name=raw.get('artist_name') or 'Unknown Artist', source=raw.get('source'), enqueued_at=time.time(), + metadata_source=raw.get('metadata_source') or 'api', ) self._items.append(item) enqueued += 1 diff --git a/core/reorganize_runner.py b/core/reorganize_runner.py index 8ef7817b..1b78e932 100644 --- a/core/reorganize_runner.py +++ b/core/reorganize_runner.py @@ -118,6 +118,7 @@ def build_runner( primary_source=item.source, strict_source=bool(item.source), stop_check=is_shutting_down_fn, + metadata_source=getattr(item, 'metadata_source', 'api') or 'api', ) return runner diff --git a/core/repair_jobs/acoustid_scanner.py b/core/repair_jobs/acoustid_scanner.py index 660c216f..00f900a1 100644 --- a/core/repair_jobs/acoustid_scanner.py +++ b/core/repair_jobs/acoustid_scanner.py @@ -254,6 +254,74 @@ class AcoustIDScannerJob(RepairJob): if title_sim >= title_threshold and artist_sim >= artist_threshold: return + # Issue #587 (Foxxify) — top recording's metadata mismatched, but + # AcoustID often returns multiple recordings per fingerprint + # (sample collisions, multi-MB-record cases). Check ALL of them + # before flagging — if any candidate's metadata matches expected + # title + artist, the file IS the right song and AcoustID's top + # match was just a wrong-credited recording. + from core.matching.acoustid_candidates import ( + duration_mismatches_strongly, + find_matching_recording, + ) + from core.matching.artist_aliases import artist_names_match + + def _scanner_title_sim(a, b): + return SequenceMatcher(None, _normalize(a), _normalize(b)).ratio() + + def _scanner_artist_sim(expected_a, actual_a): + _, score = artist_names_match(expected_a, actual_a, threshold=artist_threshold) + return score + + candidate_match, _, _ = find_matching_recording( + fp_result.get('recordings') or [], + expected['title'], + expected_artist, + title_threshold=title_threshold, + artist_threshold=artist_threshold, + similarity=_scanner_title_sim, + artist_similarity=_scanner_artist_sim, + ) + if candidate_match is not None: + # A lower-ranked candidate matched — file IS the right song. + # No finding. + if context.report_progress: + context.report_progress( + log_line=( + f'Resolved (lower-ranked candidate match): {fname} — ' + f'expected "{expected["title"]}" matched candidate ' + f'"{candidate_match.get("title")}" by ' + f'"{candidate_match.get("artist")}"' + ), + log_type='ok', + ) + return + + # Issue #587 (Foxxify "17min mashup → 5min track") — duration + # guard against fingerprint hash collisions. When the file's + # actual duration differs from AcoustID's matched recording by + # more than max(60s, 35%), the fingerprint is almost certainly + # a sample/intro collision, not a real recording match. Don't + # produce a confident "Wrong Song" finding. + try: + file_duration_s = (expected.get('duration_ms') or 0) / 1000.0 + except Exception: + file_duration_s = 0 + candidate_duration_s = best_recording.get('duration') + if candidate_duration_s is None and best_recording.get('length'): + candidate_duration_s = best_recording.get('length') + if duration_mismatches_strongly(file_duration_s, candidate_duration_s): + if context.report_progress: + context.report_progress( + log_line=( + f'Skipped (duration mismatch suggests fingerprint collision): ' + f'{fname} — expected {file_duration_s:.0f}s, AcoustID ' + f'candidate {candidate_duration_s:.0f}s' + ), + log_type='skip', + ) + return + # Mismatch detected if context.report_progress: context.report_progress( @@ -326,7 +394,8 @@ class AcoustIDScannerJob(RepairJob): t.file_path, t.track_number, al.title AS album_title, al.thumb_url, ar.thumb_url, NULLIF(t.track_artist, '') AS track_artist, - ar.name AS album_artist + ar.name AS album_artist, + t.duration FROM tracks t LEFT JOIN artists ar ON ar.id = t.artist_id LEFT JOIN albums al ON al.id = t.album_id @@ -352,6 +421,11 @@ class AcoustIDScannerJob(RepairJob): 'artist_thumb_url': row[7] or None, 'track_artist': row[8] or '', # raw (may be empty) 'album_artist': row[9] or '', + # Duration in MS (DB stores ms). Used by the + # duration-mismatch guard to spot fingerprint + # collisions where the matched recording is a + # totally different length. + 'duration_ms': row[10] or 0, } except Exception as e: logger.error("Error loading tracks from DB: %s", e) diff --git a/core/repair_jobs/orphan_file_detector.py b/core/repair_jobs/orphan_file_detector.py index 80c5a237..5c6e69cd 100644 --- a/core/repair_jobs/orphan_file_detector.py +++ b/core/repair_jobs/orphan_file_detector.py @@ -67,22 +67,28 @@ class OrphanFileDetectorJob(RepairJob): suffix = '/'.join(parts[-depth:]).lower() known_suffixes.add(suffix) - # Build title+artist sets for tag-based fallback matching + # Build title+artist sets for fallback matching. Include both + # track artist and album artist so Picard-style albumartist paths + # don't look orphaned when tracks have featured/guest artists. cursor.execute(""" - SELECT t.title, ar.name FROM tracks t - LEFT JOIN artists ar ON ar.id = t.artist_id + SELECT t.title, track_ar.name, album_ar.name FROM tracks t + LEFT JOIN artists track_ar ON track_ar.id = t.artist_id + LEFT JOIN albums al ON al.id = t.album_id + LEFT JOIN artists album_ar ON album_ar.id = al.artist_id WHERE t.title IS NOT NULL AND t.title != '' """) for row in cursor.fetchall(): title = (row[0] or '').lower().strip() - artist = (row[1] or '').lower().strip() if title: - known_titles.add((title, artist)) - # Also store normalized version (stripped of feat., parentheticals, etc.) - clean_t = _strip_extras(title) - clean_a = _strip_extras(artist) - if clean_t: - known_titles_clean.add((clean_t, clean_a)) + for artist_value in (row[1], row[2]): + artist = (artist_value or '').lower().strip() + known_titles.add((title, artist)) + # Also store normalized version (stripped of feat., + # parentheticals, etc.) + clean_t = _strip_extras(title) + clean_a = _strip_extras(artist) + if clean_t: + known_titles_clean.add((clean_t, clean_a)) except Exception as e: logger.error("Error reading known file paths from DB: %s", e, exc_info=True) result.errors += 1 @@ -144,14 +150,21 @@ class OrphanFileDetectorJob(RepairJob): if audio: file_title = ((audio.get('title') or [None])[0] or '').lower().strip() file_artist = ((audio.get('artist') or [None])[0] or '').lower().strip() + file_albumartist = ( + (audio.get('albumartist') or audio.get('album_artist') or [None])[0] + or '' + ).lower().strip() if file_title: + file_artists = [a for a in (file_artist, file_albumartist) if a] + if not file_artists: + file_artists = [''] # Exact match first (fast path) - if (file_title, file_artist) in known_titles: + if any((file_title, artist) in known_titles for artist in file_artists): is_known = True else: # Normalized match: strip (feat. X), [FLAC 16bit], etc. clean_title = _strip_extras(file_title) - clean_artist = _strip_extras(file_artist) + clean_artist = _strip_extras(file_artists[0]) # Also try first artist only (handles "Gorillaz, Dennis Hopper" → "Gorillaz") first_artist = clean_artist.split(',')[0].strip() if clean_artist else '' if clean_title and ( @@ -159,6 +172,16 @@ class OrphanFileDetectorJob(RepairJob): (first_artist and (clean_title, first_artist) in known_titles_clean) ): is_known = True + if clean_title and not is_known: + for artist in file_artists[1:]: + clean_artist = _strip_extras(artist) + first_artist = clean_artist.split(',')[0].strip() if clean_artist else '' + if ( + (clean_title, clean_artist) in known_titles_clean or + (first_artist and (clean_title, first_artist) in known_titles_clean) + ): + is_known = True + break except Exception as e: logger.debug("tag-based orphan check: %s", e) diff --git a/core/repair_jobs/unknown_artist_fixer.py b/core/repair_jobs/unknown_artist_fixer.py index bf037748..774cf32c 100644 --- a/core/repair_jobs/unknown_artist_fixer.py +++ b/core/repair_jobs/unknown_artist_fixer.py @@ -160,8 +160,18 @@ class UnknownArtistFixerJob(RepairJob): # Compute expected file path expected_rel = None if reorganize_files and corrected.get('artist') and corrected.get('album'): - from core.repair_jobs.library_reorganize import _build_path_from_template, _get_audio_quality - quality = _get_audio_quality(resolved) + # Issue #646: `core.repair_jobs.library_reorganize` was + # rewritten in commit ca5c9316 ("Rewrite Library + # Reorganize job to delegate to per-album planner") and + # the private `_build_path_from_template` / + # `_get_audio_quality` helpers moved out. They live in + # the import pipeline now — same shape, different + # module. Without this re-wire the Unknown Artist Fixer + # crashes on first scan with `ImportError: cannot + # import name '_build_path_from_template'`. + from core.imports.paths import get_file_path_from_template_raw + from core.imports.file_ops import get_audio_quality_string + quality = get_audio_quality_string(resolved) tmpl_ctx = { 'artist': corrected['artist'], 'albumartist': corrected['artist'], @@ -173,7 +183,7 @@ class UnknownArtistFixerJob(RepairJob): 'quality': quality, 'albumtype': 'Album', } - folder, fname_base = _build_path_from_template(album_template, tmpl_ctx) + folder, fname_base = get_file_path_from_template_raw(album_template, tmpl_ctx) file_ext = os.path.splitext(resolved)[1] if quality and f'[{quality}]' not in fname_base: fname_base = f"{fname_base} [{quality}]" diff --git a/core/repair_worker.py b/core/repair_worker.py index 2c0b07f8..9af2ff30 100644 --- a/core/repair_worker.py +++ b/core/repair_worker.py @@ -35,6 +35,81 @@ logger = get_logger("repair_worker") AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'} +def _album_fill_artist_names_match(expected_artist: str, candidate_artist: str) -> bool: + """Strict artist gate for Album Completeness auto-fill. + + Auto-fill moves/copies files into an existing album, so artist identity + must outrank album/title similarity. Use the alias-aware matcher when it + is available, then fall back to conservative normalized similarity. + """ + expected = (expected_artist or '').strip() + candidate = (candidate_artist or '').strip() + if not expected or not candidate: + return False + + try: + from core.matching.artist_aliases import artist_names_match + matched, score = artist_names_match(expected, candidate, threshold=0.82) + if matched: + return True + if score < 0.82: + return False + except Exception as alias_err: + logger.debug("artist_names_match unavailable, using fallback: %s", alias_err) + + try: + from core.matching_engine import MusicMatchingEngine + engine = MusicMatchingEngine() + expected_norm = engine.clean_artist(expected) + candidate_norm = engine.clean_artist(candidate) + except Exception: + expected_norm = expected.lower() + candidate_norm = candidate.lower() + + if not expected_norm or not candidate_norm: + return False + return expected_norm == candidate_norm or SequenceMatcher(None, expected_norm, candidate_norm).ratio() >= 0.82 + + +def _album_fill_target_artist_allows_track(album_artist: str, track_artists: List[str]) -> bool: + """Return whether a source track can be auto-filled into an album artist. + + Compilation-style album artists are allowed to contain varied track + artists. Normal albums require at least one source track artist to match + the target album artist before any local candidate is considered. + """ + album_artist = (album_artist or '').strip() + if not album_artist: + return False + + normalized_album_artist = album_artist.lower().strip() + if normalized_album_artist in {'various artists', 'various', 'soundtrack'}: + return True + + source_artists = [str(a).strip() for a in (track_artists or []) if str(a or '').strip()] + if not source_artists: + return True + + return any(_album_fill_artist_names_match(album_artist, artist) for artist in source_artists) + + +def _split_acoustid_credit(credit: str) -> List[str]: + """Split an AcoustID artist credit into individual contributor names. + + Reuses the matching layer's credit splitter so the AcoustID retag + path tags multi-artist tracks the same way the post-download + enrichment pipeline does (comma / ampersand / feat. / etc). + Returns ``[credit]`` for single-artist credits — the writer's + ``len > 1`` check is what gates whether the multi-value tag gets + written. + """ + try: + from core.matching.artist_aliases import split_artist_credit + return split_artist_credit(credit) + except Exception: + return [credit] if credit else [] + + def _resolve_file_path(file_path, transfer_folder, download_folder=None, config_manager=None, plex_client=None): """Resolve a stored DB path to an actual file on disk. @@ -1715,6 +1790,17 @@ class RepairWorker: tag_updates = {'title': aid_title} if aid_artist: tag_updates['artist_name'] = aid_artist + # Issue #587 — derive a per-artist list from + # AcoustID's credit string when it carries + # multiple contributors. The post-download + # enrichment pipeline preserves multi-value + # ARTISTS tags via the user's + # `write_multi_artist` setting; the repair + # path was bypassing that and writing a + # single-string TPE1 only. Now respects the + # same setting via the writer's new + # `artists_list` derivation. + tag_updates['artists_list'] = _split_acoustid_credit(aid_artist) write_tags_to_file(resolved, tag_updates) logger.info("Wrote corrected tags to file: %s", resolved) except Exception as tag_err: @@ -2069,6 +2155,21 @@ class RepairWorker: track_details.append({'track': track_name, 'status': 'skipped', 'reason': 'no track name'}) continue + if not _album_fill_target_artist_allows_track(artist_name, track_artists): + skipped_count += 1 + logger.warning( + "Album auto-fill skipped '%s': source artist(s) %s do not match target album artist '%s'", + track_name, track_artists, artist_name, + ) + track_details.append({ + 'track': track_name, + 'status': 'skipped', + 'reason': 'source artist does not match target album artist', + 'source_artists': track_artists, + 'target_artist': artist_name, + }) + continue + # Search library for this track candidates = self.db.search_tracks(title=track_name, artist=artist_search, limit=20) @@ -2089,8 +2190,22 @@ class RepairWorker: # Artist match (more lenient) cand_artist = getattr(cand, 'artist_name', '') or '' - artist_sim = SequenceMatcher(None, artist_search.lower(), cand_artist.lower()).ratio() - if artist_sim < 0.50: + candidate_artist_fields = [ + cand_artist, + getattr(cand, 'track_artist', '') or '', + ] + expected_artist_names = track_artists or [artist_name] + if not any( + _album_fill_artist_names_match(expected, candidate) + for expected in expected_artist_names + for candidate in candidate_artist_fields + ): + logger.debug( + "Album auto-fill rejected candidate '%s' by '%s' for expected artist(s) %s", + getattr(cand, 'title', ''), + cand_artist, + expected_artist_names, + ) continue # Quality gate diff --git a/core/search/orchestrator.py b/core/search/orchestrator.py index da934759..a1961424 100644 --- a/core/search/orchestrator.py +++ b/core/search/orchestrator.py @@ -31,7 +31,7 @@ from . import sources logger = logging.getLogger(__name__) VALID_SOURCES = ( - 'spotify', 'itunes', 'deezer', 'discogs', 'hydrabase', 'musicbrainz', + 'spotify', 'itunes', 'deezer', 'discogs', 'hydrabase', 'musicbrainz', 'amazon', ) VALID_STREAM_SOURCES = VALID_SOURCES + ('youtube_videos',) @@ -88,6 +88,13 @@ def resolve_client(source_name: str, deps: SearchDeps) -> tuple[Any, bool]: except Exception as e: logger.warning(f"MusicBrainz search client init failed: {e}") return None, False + if source_name == 'amazon': + try: + from core.metadata.registry import get_amazon_client + return get_amazon_client(), True + except Exception as e: + logger.warning(f"Amazon Music client init failed: {e}") + return None, False return None, False @@ -183,6 +190,8 @@ def _alternate_sources(primary_source: str, deps: SearchDeps) -> list[str]: alts.append('discogs') if primary_source != 'hydrabase' and hydrabase_available: alts.append('hydrabase') + if primary_source != 'amazon': + alts.append('amazon') # always available (T2Tunes, no auth) alts.append('youtube_videos') # always available (yt-dlp, no auth) alts.append('musicbrainz') # always available (public API) return alts diff --git a/core/seasonal_discovery.py b/core/seasonal_discovery.py index 159776f7..b813717a 100644 --- a/core/seasonal_discovery.py +++ b/core/seasonal_discovery.py @@ -707,7 +707,8 @@ class SeasonalDiscoveryService: artist_name, album_cover_url, release_date, - popularity + popularity, + source FROM seasonal_albums WHERE season_key = ? AND source = ? ORDER BY popularity DESC, album_name ASC diff --git a/core/soulseek_client.py b/core/soulseek_client.py index 6b928688..8a7f82dd 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -18,7 +18,13 @@ from core.download_plugins.types import ( SearchResult, TrackResult, ) +from core.download_plugins.album_bundle import ( + copy_audio_files_atomically, + get_poll_interval, + get_poll_timeout, +) from core.download_plugins.base import DownloadSourcePlugin +from utils.async_helpers import run_async logger = get_logger("soulseek_client") @@ -250,6 +256,7 @@ class SoulseekClient(DownloadSourcePlugin): if response.status in [200, 201, 204]: # Accept 200 OK, 201 Created, and 204 No Content self._last_401_logged = False # Reset on success + self._last_unreachable_logged = False # Same reset for unreachable-host suppression try: if response_text.strip(): # Only parse if there's content return await response.json() @@ -291,6 +298,25 @@ class SoulseekClient(DownloadSourcePlugin): f"{method} {url} — slskd may be overloaded or unreachable" ) return None + except aiohttp.ClientConnectorError as e: + # Issue #649: slskd_url is configured but the host is unreachable + # (slskd not running, wrong port, DNS / Docker bridge issue). + # Status polling at /api/downloads/status fans out to every plugin + # including soulseek even when the user has soulseek toggled out + # of their active download sources, so each frontend poll + # produced an ERROR log line — visible spam during any + # non-soulseek download. Suppress repeats to debug; emit one + # WARNING with actionable context, then reset on any successful + # response (slskd came back up). + if not getattr(self, '_last_unreachable_logged', False): + logger.warning( + f"slskd unreachable at {self.base_url}: {e}. " + f"Either start slskd or clear `soulseek.slskd_url` in settings " + f"if you don't use Soulseek. Suppressing further connection errors." + ) + self._last_unreachable_logged = True + logger.debug(f"slskd connection failed: {method} {url}: {e}") + return None except Exception as e: logger.error(f"Error making API request: {e}") return None @@ -348,6 +374,19 @@ class SoulseekClient(DownloadSourcePlugin): f"{method} {url} — slskd may be overloaded or unreachable" ) return None + except aiohttp.ClientConnectorError as e: + # Issue #649 — same suppression as _make_request. Direct + # request is a less common path but uses the same base_url, + # so the same unreachable-host condition fires here. + if not getattr(self, '_last_unreachable_logged', False): + logger.warning( + f"slskd unreachable at {self.base_url}: {e}. " + f"Either start slskd or clear `soulseek.slskd_url` in settings " + f"if you don't use Soulseek. Suppressing further connection errors." + ) + self._last_unreachable_logged = True + logger.debug(f"slskd direct connection failed: {method} {url}: {e}") + return None except Exception as e: logger.error(f"Error making direct API request: {e}") return None @@ -1423,6 +1462,326 @@ class SoulseekClient(DownloadSourcePlugin): logger.info(f"Downloading: {best_result.filename} ({quality_info}) from {best_result.username}") return await self.download(best_result.username, best_result.filename, best_result.size) + + def download_album_to_staging( + self, + album_name: str, + artist_name: str, + staging_dir: str, + progress_callback=None, + *, + preferred_source: Optional[Dict[str, Any]] = None, + preferred_tracks: Optional[List[TrackResult]] = None, + ) -> Dict[str, Any]: + """One-shot Soulseek album download. + + Search for one album folder, enqueue files from that single + ``username + folder_path``, wait for slskd to report completion, + then copy completed files into the private album-bundle staging + directory. If the folder cannot be selected or enqueued cleanly, + callers may fall back to the existing per-track Soulseek flow. + Once files are staged, the per-track staging matcher owns final + import, same as torrent / usenet album bundles. + """ + result: Dict[str, Any] = { + 'success': False, + 'files': [], + 'error': None, + 'fallback': True, + 'partial': False, + } + if not self.is_configured(): + result['error'] = 'Soulseek source not configured' + return result + + def _emit(state: str, **extra) -> None: + if progress_callback: + try: + progress_callback({'state': state, **extra}) + except Exception as cb_exc: + logger.debug("[Soulseek album] progress callback failed: %s", cb_exc) + + picked = None + folder_tracks = list(preferred_tracks or []) + username = (preferred_source or {}).get('username', '') if preferred_source else '' + folder_path = (preferred_source or {}).get('folder_path', '') if preferred_source else '' + if username and folder_path: + logger.info( + "[Soulseek album] Using preflight-selected folder %s:%s", + username, + folder_path, + ) + _emit('searching', query=f"{artist_name} {album_name}".strip(), release=folder_path) + else: + query = f"{artist_name} {album_name}".strip() + _emit('searching', query=query) + try: + _, albums = run_async(self.search(query, timeout=30)) + except Exception as exc: + result['error'] = f'Soulseek album search failed: {exc}' + return result + + if not albums: + result['error'] = 'No complete Soulseek album folders found' + return result + + picked = self._pick_album_bundle_folder(albums, album_name, artist_name) + if picked is None: + result['error'] = 'No suitable Soulseek album folder after filtering' + return result + + folder_path = getattr(picked, 'album_path', '') or '' + username = getattr(picked, 'username', '') or '' + if not username or not folder_path: + result['error'] = 'No suitable Soulseek album folder after filtering' + return result + + logger.info( + "[Soulseek album] Picked %s:%s (%s tracks, quality=%s)", + username, + folder_path, + getattr(picked, 'track_count', 0), + getattr(picked, 'dominant_quality', ''), + ) + _emit( + 'queued', + release=getattr(picked, 'album_title', folder_path) if picked else folder_path, + count=getattr(picked, 'track_count', 0) if picked else len(folder_tracks), + ) + + if not folder_tracks: + try: + browse_files = run_async(self.browse_user_directory(username, folder_path)) + except Exception as exc: + result['error'] = f'Soulseek folder browse failed: {exc}' + return result + + if not browse_files: + result['error'] = 'Could not browse selected Soulseek album folder' + return result + + folder_tracks = self.parse_browse_results_to_tracks( + username, + browse_files, + directory=folder_path, + ) + folder_tracks = self.filter_results_by_quality_preference(folder_tracks) + if not folder_tracks: + result['error'] = 'Selected Soulseek album folder contained no audio files' + return result + + transfer_keys: Dict[tuple, TrackResult] = {} + _emit( + 'downloading', + release=getattr(picked, 'album_title', folder_path) if picked else folder_path, + count=len(folder_tracks), + ) + for track in folder_tracks: + try: + download_id = run_async(self.download(track.username, track.filename, track.size)) + except Exception as exc: + logger.warning("[Soulseek album] Failed to enqueue %s: %s", track.filename, exc) + continue + if download_id: + transfer_keys[(track.username, track.filename)] = track + + if not transfer_keys: + result['error'] = 'No Soulseek album files could be enqueued' + return result + + result['fallback'] = False + completed = self._poll_album_bundle_downloads(transfer_keys, _emit) + if not completed: + result['error'] = 'Soulseek album download failed or timed out' + return result + + _emit('staging', release=getattr(picked, 'album_title', folder_path) if picked else folder_path) + copied = copy_audio_files_atomically(completed, Path(staging_dir)) + if not copied: + result['error'] = 'No Soulseek album files copied to staging' + return result + + partial = len(copied) < len(transfer_keys) + if partial: + logger.warning( + "[Soulseek album] Staged partial album for '%s': %d/%d files", + album_name, + len(copied), + len(transfer_keys), + ) + else: + logger.info("[Soulseek album] Staged %d files for '%s'", len(copied), album_name) + _emit('staged', count=len(copied)) + result['success'] = True + result['files'] = copied + result['partial'] = partial + result['expected_count'] = len(transfer_keys) + result['completed_count'] = len(copied) + return result + + def _pick_album_bundle_folder( + self, + albums: List[AlbumResult], + album_name: str, + artist_name: str, + ) -> Optional[AlbumResult]: + scored = [] + for album in albums: + tracks = self.filter_results_by_quality_preference(list(getattr(album, 'tracks', []) or [])) + if not tracks: + continue + album_text = f"{getattr(album, 'album_title', '')} {getattr(album, 'album_path', '')}" + artist_text = f"{getattr(album, 'artist', '')} {getattr(album, 'album_path', '')}" + album_score = self._bundle_similarity(album_name, album_text) + artist_score = self._bundle_similarity(artist_name, artist_text) + track_count = int(getattr(album, 'track_count', 0) or len(tracks)) + count_score = 1.0 if track_count >= 3 else 0.35 + score = ( + album_score * 0.42 + + artist_score * 0.22 + + count_score * 0.12 + + min(1.0, len(tracks) / max(1, track_count)) * 0.12 + + float(getattr(album, 'quality_score', 0.0) or 0.0) * 0.12 + ) + scored.append((score, len(tracks), album)) + if not scored: + return None + scored.sort(key=lambda row: (row[0], row[1], getattr(row[2], 'quality_score', 0.0)), reverse=True) + best_score, _, best = scored[0] + if best_score < 0.58: + logger.warning("[Soulseek album] Best folder score %.3f below threshold", best_score) + return None + return best + + @staticmethod + def _bundle_similarity(expected: Any, actual: Any) -> float: + import re + from difflib import SequenceMatcher + left = re.sub(r'[^a-z0-9]+', ' ', str(expected or '').lower()).strip() + right = re.sub(r'[^a-z0-9]+', ' ', str(actual or '').lower()).strip() + if not left or not right: + return 0.0 + if left == right: + return 1.0 + left_words = set(left.split()) + right_words = set(right.split()) + if left_words and left_words.issubset(right_words): + return 0.92 + if right_words and right_words.issubset(left_words): + return 0.86 + if left in right or right in left: + return min(len(left), len(right)) / max(len(left), len(right)) + return SequenceMatcher(None, left, right).ratio() + + def _poll_album_bundle_downloads(self, transfer_keys: Dict[tuple, TrackResult], emit) -> List[Path]: + deadline = time.monotonic() + get_poll_timeout() + interval = get_poll_interval() + completed_paths: Dict[tuple, Path] = {} + failed_states: Dict[tuple, str] = {} + while time.monotonic() < deadline: + try: + downloads = run_async(self.get_all_downloads()) + except Exception as exc: + logger.warning("[Soulseek album] Poll error: %s", exc) + downloads = [] + + by_key = {} + for dl in downloads: + exact_key = (dl.username, dl.filename) + by_key[exact_key] = dl + basename_key = ( + dl.username, + os.path.basename((dl.filename or '').replace('\\', '/')), + ) + by_key.setdefault(basename_key, dl) + for key, track in transfer_keys.items(): + if key in completed_paths or key in failed_states: + continue + dl = by_key.get(key) or by_key.get(( + key[0], + os.path.basename((key[1] or '').replace('\\', '/')), + )) + state = (getattr(dl, 'state', '') or '') if dl else '' + if any(token in state for token in ('Errored', 'Failed', 'Rejected', 'TimedOut')): + failed_states[key] = state or 'Failed' + logger.warning( + "[Soulseek album] Transfer failed from selected folder: %s (%s)", + os.path.basename((track.filename or '').replace('\\', '/')), + failed_states[key], + ) + continue + if dl and ('Completed' in state or 'Succeeded' in state): + if dl.size and dl.transferred and dl.transferred < dl.size: + continue + path = self._resolve_downloaded_album_file(track.filename) + if path: + completed_paths[key] = path + else: + logger.debug( + "[Soulseek album] Transfer completed but local file not found yet: %s", + track.filename, + ) + emit( + 'downloading', + progress=round(len(completed_paths) / max(1, len(transfer_keys)) * 100, 1), + count=len(completed_paths), + failed=len(failed_states), + ) + if completed_paths and len(completed_paths) + len(failed_states) == len(transfer_keys): + logger.warning( + "[Soulseek album] Selected folder finished with %d completed and %d failed transfer(s)", + len(completed_paths), + len(failed_states), + ) + return list(completed_paths.values()) + if not completed_paths and len(failed_states) == len(transfer_keys): + logger.warning("[Soulseek album] All %d transfer(s) failed from selected folder", len(failed_states)) + return [] + if len(completed_paths) == len(transfer_keys): + return list(completed_paths.values()) + time.sleep(interval) + pending = len(transfer_keys) - len(completed_paths) - len(failed_states) + if completed_paths: + logger.warning( + "[Soulseek album] Timed out with partial album: %d completed, %d failed, %d pending", + len(completed_paths), + len(failed_states), + pending, + ) + return list(completed_paths.values()) + logger.error( + "[Soulseek album] Timed out waiting for %d album files (%d failed, %d pending)", + len(transfer_keys), + len(failed_states), + pending, + ) + return [] + + def _resolve_downloaded_album_file(self, remote_filename: str) -> Optional[Path]: + basename = os.path.basename((remote_filename or '').replace('\\', '/')) + if not basename: + return None + candidates = [ + self.download_path / remote_filename, + self.download_path / basename, + ] + normalized_parts = [p for p in remote_filename.replace('\\', '/').split('/') if p] + if normalized_parts: + candidates.append(self.download_path.joinpath(*normalized_parts)) + for candidate in candidates: + try: + if candidate.exists() and candidate.is_file(): + return candidate + except OSError: + continue + try: + matches = list(self.download_path.rglob(basename)) + except OSError: + matches = [] + for match in matches: + if match.is_file(): + return match + return None async def check_connection(self) -> bool: """Check if slskd is running and connected to the Soulseek network""" @@ -1464,14 +1823,71 @@ class SoulseekClient(DownloadSourcePlugin): 'other': (0, 500), } + def _drop_quarantined_sources(self, results: List[TrackResult]) -> List[TrackResult]: + """Filter out candidates whose `(username, filename)` is on the + quarantine record. Issue #652. + + Reads quarantine sidecars fresh each call so newly-quarantined + sources are honored immediately on the next search — no client + state to invalidate. Filesystem cost is bounded (one listdir + + N small JSON reads) and dwarfed by the Soulseek search itself. + + Returns the input list unchanged when the quarantine directory + is absent, empty, or unreadable — i.e. defaults to today's + behaviour if anything goes wrong on the dedup path. + """ + try: + from core.imports.quarantine import get_quarantined_source_keys + download_path = config_manager.get('soulseek.download_path', './downloads') + quarantine_dir = os.path.join(download_path, 'ss_quarantine') + blocked = get_quarantined_source_keys(quarantine_dir) + except Exception as exc: + logger.debug("quarantine dedup: failed to load source keys, skipping filter: %s", exc) + return results + + if not blocked: + return results + + kept: List[TrackResult] = [] + skipped = 0 + for candidate in results: + key = (candidate.username or '', candidate.filename or '') + if key in blocked: + skipped += 1 + continue + kept.append(candidate) + + if skipped: + logger.info( + f"Quarantine dedup: dropped {skipped} candidate(s) matching previously-quarantined sources; " + f"{len(kept)} remain" + ) + return kept + def filter_results_by_quality_preference(self, results: List[TrackResult]) -> List[TrackResult]: """ Filter candidates based on user's quality profile with bitrate density constraints. Uses priority waterfall logic: tries highest priority quality first, falls back to lower priorities. Returns candidates matching quality profile constraints, sorted by confidence and effective bitrate. + + Issue #652: also drops candidates whose `(username, filename)` + matches a previously-quarantined download. Without this pre-filter + the auto-wishlist processor's ranking is deterministic — the same + `(uploader, file)` keeps winning the quality picker, downloading, + failing AcoustID, quarantining, and re-queueing in an infinite + loop. Users wake up to hundreds of duplicate `.quarantined` files + for the same source URL. """ from database.music_database import MusicDatabase + if not results: + return [] + + # Drop sources already quarantined — bypass the quality picker + # entirely so the same bad upload doesn't get re-selected on the + # next wishlist cycle. Filesystem read is bounded (~few hundred + # sidecars in practical use × <1ms each). + results = self._drop_quarantined_sources(results) if not results: return [] diff --git a/core/sync/__init__.py b/core/sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/sync/match_overrides.py b/core/sync/match_overrides.py new file mode 100644 index 00000000..f83c007d --- /dev/null +++ b/core/sync/match_overrides.py @@ -0,0 +1,119 @@ +"""Sync match overrides — user-confirmed source→server track pairings. + +When a user picks a local file via "Find & Add" on the Server Playlist +compare view, that selection should persist as a hard match across +future syncs — bypassing the fuzzy/exact title-match algorithm +entirely. This module provides pure helpers that the web layer calls +to resolve and persist those overrides through the existing +`sync_match_cache` table. + +Override semantics: + - One mapping per (source_track_id, server_source). UNIQUE + constraint on the table enforces single mapping per pair. + - Stored with confidence=1.0 to distinguish from auto-discovered + matches (which use the actual title-similarity score). + - Read at the START of the matching algorithm — before pass-1 + exact and pass-2 fuzzy. Skipped sources don't re-enter the + normal matching pool. + - Stale-cache safe: if the cached server_track_id doesn't exist + in the current server_tracks list (track removed from server), + the override is silently skipped and normal matching runs. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List, Optional + + +def resolve_match_overrides( + source_tracks: List[Dict[str, Any]], + server_tracks: List[Dict[str, Any]], + cache_lookup: Callable[[str], Optional[Any]], +) -> Dict[int, int]: + """Map source-track indexes to server-track indexes for cached overrides. + + Pure function. `cache_lookup(source_track_id) -> server_track_id or + None` is injected by the caller (web layer wraps the DB call). + + Returns ``{source_idx: server_idx}``. Only includes pairs where: + - source_track has a non-empty `source_track_id` + - cache_lookup returns a server_track_id + - that server_track_id exists in server_tracks (no stale cache + entries pointing at deleted tracks) + - the server_track hasn't already been claimed by an earlier + override (defensive — UNIQUE on the cache table prevents this + in practice) + + Caller uses the returned dict to short-circuit the per-source + matching loop: indices in the dict skip the exact/fuzzy passes. + """ + if not source_tracks or not server_tracks: + return {} + + server_id_to_idx: Dict[str, int] = {} + for j, svr in enumerate(server_tracks): + sid = svr.get("id") if isinstance(svr, dict) else None + if sid is not None: + key = str(sid) + if key not in server_id_to_idx: + server_id_to_idx[key] = j + + overrides: Dict[int, int] = {} + used_server: set[int] = set() + + for i, src in enumerate(source_tracks): + if not isinstance(src, dict): + continue + src_id = src.get("source_track_id") + if not src_id: + continue + try: + cached_server_id = cache_lookup(str(src_id)) + except Exception: + cached_server_id = None + if not cached_server_id: + continue + j = server_id_to_idx.get(str(cached_server_id)) + if j is None or j in used_server: + continue + overrides[i] = j + used_server.add(j) + + return overrides + + +def record_manual_match( + db: Any, + source_track_id: str, + server_source: str, + server_track_id: Any, + server_track_title: str = "", + source_title: str = "", + source_artist: str = "", +) -> bool: + """Persist a user-confirmed source→server pairing as a hard override. + + Wraps `db.save_sync_match_cache` with confidence=1.0 (the manual + match marker). Normalized title/artist fields are informational + only — the cache is keyed by `(spotify_track_id, server_source)`, + so the normalization is just for inspection and future debugging. + + Returns True on persist success, False on any failure (DB, missing + args, etc). Never raises. + """ + if not source_track_id or not server_source or server_track_id is None: + return False + if not hasattr(db, "save_sync_match_cache"): + return False + try: + return bool(db.save_sync_match_cache( + spotify_track_id=str(source_track_id), + normalized_title=(source_title or "").lower().strip(), + normalized_artist=(source_artist or "").lower().strip(), + server_source=server_source, + server_track_id=server_track_id, + server_track_title=server_track_title or "", + confidence=1.0, + )) + except Exception: + return False diff --git a/core/tag_writer.py b/core/tag_writer.py index 11cb5be1..5b4d5752 100644 --- a/core/tag_writer.py +++ b/core/tag_writer.py @@ -299,18 +299,31 @@ def write_tags_to_file(file_path: str, db_data: Dict[str, Any], elif isinstance(genres, str): genre_str = genres + # Multi-value artist support — issue #587. Caller can pass + # `artists_list` (per-track list of contributor names) and the + # writer respects the user's `metadata_enhancement.tags.write_multi_artist` + # config the same way the post-download enrichment pipeline does. + # When the setting is on AND the list has >1 entry: + # - ID3 keeps TPE1 as the joined display string (already in `artist`) + # and writes a separate TXXX:Artists frame with the list + # - Vorbis writes an `artists` multi-value key alongside `artist` + # - MP4 writes \xa9ART as the list when on, single string when off + # When OFF or the list is empty/single — same single-string write + # as before. Backward compatible for callers that don't pass it. + artists_list = _resolve_artists_list_for_write(db_data) + if isinstance(audio.tags, ID3): written = _write_id3(audio, title, artist, album_artist, album, year, genre_str, track_num, total_tracks, - disc_num, bpm) + disc_num, bpm, artists_list=artists_list) elif isinstance(audio, (FLAC, OggVorbis)) or type(audio).__name__ == 'OggOpus': written = _write_vorbis(audio, title, artist, album_artist, album, year, genre_str, track_num, total_tracks, - disc_num, bpm) + disc_num, bpm, artists_list=artists_list) elif isinstance(audio, MP4): written = _write_mp4(audio, title, artist, album_artist, album, year, genre_str, track_num, total_tracks, - disc_num, bpm) + disc_num, bpm, artists_list=artists_list) # Embed cover art if requested if embed_cover: @@ -343,8 +356,43 @@ def write_tags_to_file(file_path: str, db_data: Dict[str, Any], # ── Format-specific writers ── + +def _resolve_artists_list_for_write(db_data: Dict[str, Any]) -> Optional[List[str]]: + """Pull a multi-value artists list from db_data when caller supplied one. + + Accepts either ``artists_list`` (list of names) or ``artists`` (same + shape — kept for symmetry with the post-process pipeline's + ``_artists_list`` field). Drops empty / non-string entries. Returns + ``None`` when no list was supplied so format writers can branch on + "single-string only" vs "multi-value too". + """ + raw = db_data.get('artists_list') or db_data.get('artists') or db_data.get('_artists_list') + if not raw: + return None + if not isinstance(raw, (list, tuple)): + return None + cleaned = [] + for entry in raw: + if isinstance(entry, str): + text = entry.strip() + if text: + cleaned.append(text) + return cleaned or None + + +def _multi_artist_write_enabled() -> bool: + """Read the same config flag the enrichment pipeline reads, so the + repair-path retag respects the user's choice.""" + try: + from config.settings import config_manager + return bool(config_manager.get('metadata_enhancement.tags.write_multi_artist', False)) + except Exception: + return False + + def _write_id3(audio, title, artist, album_artist, album, year, genre, - track_num, total_tracks, disc_num, bpm) -> List[str]: + track_num, total_tracks, disc_num, bpm, + artists_list: Optional[List[str]] = None) -> List[str]: written = [] if title: audio.tags.delall('TIT2') @@ -354,6 +402,16 @@ def _write_id3(audio, title, artist, album_artist, album, year, genre, audio.tags.delall('TPE1') audio.tags.add(TPE1(encoding=3, text=[artist])) written.append('artist') + # TPE1 stays as the joined display string. When the caller + # supplied a multi-value list AND the user has the + # write_multi_artist setting on, ALSO write the per-artist + # list to a TXXX:Artists frame (Picard convention). Mirrors + # the post-download enrichment writer at + # core/metadata/enrichment.py. + if artists_list and len(artists_list) > 1 and _multi_artist_write_enabled(): + audio.tags.delall('TXXX:Artists') + audio.tags.add(TXXX(encoding=3, desc='Artists', text=list(artists_list))) + written.append('artists_multi') if album_artist: audio.tags.delall('TPE2') audio.tags.add(TPE2(encoding=3, text=[album_artist])) @@ -387,7 +445,8 @@ def _write_id3(audio, title, artist, album_artist, album, year, genre, def _write_vorbis(audio, title, artist, album_artist, album, year, genre, - track_num, total_tracks, disc_num, bpm) -> List[str]: + track_num, total_tracks, disc_num, bpm, + artists_list: Optional[List[str]] = None) -> List[str]: written = [] if title: audio['title'] = [title] @@ -395,6 +454,13 @@ def _write_vorbis(audio, title, artist, album_artist, album, year, genre, if artist: audio['artist'] = [artist] written.append('artist') + # Vorbis-style multi-value: write the per-artist list to the + # `artists` key (separate from `artist`, picard convention) when + # the caller supplied a list AND the user has multi-value write + # enabled. Mirrors enrichment.py. + if artists_list and len(artists_list) > 1 and _multi_artist_write_enabled(): + audio['artists'] = list(artists_list) + written.append('artists_multi') if album_artist: audio['albumartist'] = [album_artist] written.append('album_artist') @@ -421,14 +487,24 @@ def _write_vorbis(audio, title, artist, album_artist, album, year, genre, def _write_mp4(audio, title, artist, album_artist, album, year, genre, - track_num, total_tracks, disc_num, bpm) -> List[str]: + track_num, total_tracks, disc_num, bpm, + artists_list: Optional[List[str]] = None) -> List[str]: written = [] if title: audio['\xa9nam'] = [title] written.append('title') if artist: - audio['\xa9ART'] = [artist] - written.append('artist') + # MP4 \xa9ART can carry a list directly. When caller supplied + # a multi-value list AND user has multi-value write enabled, + # write the list. Otherwise single-string. Mirrors enrichment.py + # MP4 path. + if artists_list and len(artists_list) > 1 and _multi_artist_write_enabled(): + audio['\xa9ART'] = list(artists_list) + written.append('artist') + written.append('artists_multi') + else: + audio['\xa9ART'] = [artist] + written.append('artist') if album_artist: audio['aART'] = [album_artist] written.append('album_artist') diff --git a/core/tidal_download_client.py b/core/tidal_download_client.py index 45bdd4b9..6ae2027e 100644 --- a/core/tidal_download_client.py +++ b/core/tidal_download_client.py @@ -95,6 +95,16 @@ HLS_MAP_TAG_RE = re.compile(r'#EXT-X-MAP:.*URI="([^"]+)"') from core.download_plugins.base import DownloadSourcePlugin +def _looks_like_json_decode_error(exc: Exception) -> bool: + name = exc.__class__.__name__.lower() + message = str(exc).lower() + return ( + "jsondecodeerror" in name + or "expecting value" in message + or "could not decode json" in message + ) + + class TidalDownloadClient(DownloadSourcePlugin): """ Tidal download client using tidalapi. @@ -230,6 +240,18 @@ class TidalDownloadClient(DownloadSourcePlugin): return {'status': 'error', 'message': 'Auth completed but session invalid'} except Exception as e: + if _looks_like_json_decode_error(e): + logger.error( + "Tidal device auth check received a non-JSON response from Tidal's token endpoint: %s", + e, + ) + return { + 'status': 'error', + 'message': ( + "Tidal returned an invalid auth response while SoulSync was finishing login. " + "Try again after disabling VPN/proxy/network filtering, then restart SoulSync if it repeats." + ), + } logger.error(f"Tidal device auth check error: {e}") return {'status': 'error', 'message': str(e)} @@ -278,6 +300,27 @@ class TidalDownloadClient(DownloadSourcePlugin): return False return True + @classmethod + def _track_matches_qualifiers(cls, track, qualifiers: List[str]) -> bool: + """Issue #589 — qualifier check must inspect both track.name AND + track.album.name. For MTV Unplugged-style releases the live / + unplugged signal lives in the album title, not the track title. + A track passes if every required qualifier appears as a whole + word in either the track name OR its album name. + """ + if not qualifiers: + return True + track_name = (getattr(track, 'name', '') or '').lower() + album = getattr(track, 'album', None) + album_name = (getattr(album, 'name', '') or '').lower() if album else '' + haystack = f"{track_name} {album_name}".strip() + if not haystack: + return False + for kw in qualifiers: + if not re.search(r'\b' + re.escape(kw) + r'\b', haystack): + return False + return True + @staticmethod def _generate_shortened_queries(original: str) -> List[str]: variants: List[str] = [] @@ -342,25 +385,35 @@ class TidalDownloadClient(DownloadSourcePlugin): found = await loop.run_in_executor(None, _search) if found: + # Issue #589 — qualifier filter applies to ALL + # search attempts, not just fallbacks. If the + # primary query carries "live" / "unplugged" / + # etc, the studio cut should never be accepted + # just because Tidal returned it first. The + # filter inspects both track.name AND + # track.album.name (the live signal often lives + # in the album title for concert releases). is_fallback = attempt_idx > 0 - if is_fallback and required_qualifiers: + if required_qualifiers: filtered = [ t for t in found - if self._track_name_contains_qualifiers(getattr(t, 'name', ''), required_qualifiers) + if self._track_matches_qualifiers(t, required_qualifiers) ] if filtered: tidal_tracks = filtered successful_query = attempt_query logger.info( - f"Tidal fallback kept {len(filtered)}/{len(found)} tracks " - f"after qualifier filter {required_qualifiers} for '{attempt_query}'" + f"Tidal {'fallback' if is_fallback else 'primary'} kept " + f"{len(filtered)}/{len(found)} tracks after qualifier filter " + f"{required_qualifiers} for '{attempt_query}'" ) break else: any_fallback_filtered_out = True logger.debug( - f"Tidal fallback '{attempt_query}' returned {len(found)} tracks " - f"but none matched original qualifiers {required_qualifiers} — " + f"Tidal {'fallback' if is_fallback else 'primary'} " + f"'{attempt_query}' returned {len(found)} tracks but none " + f"matched required qualifiers {required_qualifiers} — " f"trying next variant" ) if attempt_idx < len(queries_to_try) - 1: diff --git a/core/torrent_clients/__init__.py b/core/torrent_clients/__init__.py new file mode 100644 index 00000000..ce23698c --- /dev/null +++ b/core/torrent_clients/__init__.py @@ -0,0 +1,56 @@ +"""Torrent client adapters. + +Each adapter wraps one BitTorrent client (qBittorrent, Transmission, +Deluge) behind the ``TorrentClientAdapter`` Protocol so the rest of +SoulSync can talk to whichever client the user picked through one +uniform surface. + +The active adapter is selected at runtime by the ``torrent_client.type`` +config key. See ``get_active_adapter()`` for the factory. +""" + +from __future__ import annotations + +from typing import Optional + +from config.settings import config_manager + +from core.torrent_clients.base import TorrentClientAdapter, TorrentStatus +from core.torrent_clients.deluge import DelugeAdapter +from core.torrent_clients.qbittorrent import QBittorrentAdapter +from core.torrent_clients.transmission import TransmissionAdapter + +__all__ = [ + "TorrentClientAdapter", + "TorrentStatus", + "QBittorrentAdapter", + "TransmissionAdapter", + "DelugeAdapter", + "get_active_adapter", + "adapter_for_type", +] + + +def adapter_for_type(client_type: str) -> Optional[TorrentClientAdapter]: + """Build a fresh adapter instance for the given client type string. + + ``None`` for unknown types so callers can present a helpful error + rather than crashing on a typo'd config value. + """ + if client_type == "qbittorrent": + return QBittorrentAdapter() + if client_type == "transmission": + return TransmissionAdapter() + if client_type == "deluge": + return DelugeAdapter() + return None + + +def get_active_adapter() -> Optional[TorrentClientAdapter]: + """Return an adapter for whichever torrent client the user has + selected in Settings. Reads ``torrent_client.type`` each call so a + settings change is picked up without restarting the process.""" + client_type = (config_manager.get('torrent_client.type', '') or '').strip().lower() + if not client_type: + return None + return adapter_for_type(client_type) diff --git a/core/torrent_clients/base.py b/core/torrent_clients/base.py new file mode 100644 index 00000000..4efe936e --- /dev/null +++ b/core/torrent_clients/base.py @@ -0,0 +1,110 @@ +"""Torrent client adapter contract. + +``TorrentClientAdapter`` is a structural Protocol — any class with +these method signatures is treated as a valid adapter. The download +plugin layer (built in a later commit) dispatches generically against +this surface so it doesn't have to know whether the user picked +qBittorrent, Transmission, or Deluge. + +The contract intentionally hides protocol-specific details: +- qBittorrent uses cookie auth + multipart form uploads. +- Transmission uses an X-Transmission-Session-Id header + JSON RPC. +- Deluge 2.x uses /json with a session cookie. + +All three converge on the same eight verbs below. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Protocol, runtime_checkable + + +@dataclass +class TorrentStatus: + """Adapter-uniform view of one torrent's live state. + + Field semantics: + - ``state`` is one of: ``queued`` | ``downloading`` | ``seeding`` | + ``paused`` | ``stalled`` | ``error`` | ``completed``. Each + adapter maps its native state names to this set. + - ``progress`` is 0.0–1.0. + - ``save_path`` is where files land on the torrent client's host. + For remote clients this is a path on the *remote* machine. + - ``files`` is the list of relative paths inside the torrent. Empty + until the client has finished fetching the metadata. + """ + + id: str # torrent hash (qBit, Deluge) or numeric id (Transmission) + name: str + state: str + progress: float + size: int # total size in bytes + downloaded: int # bytes downloaded so far + download_speed: int # bytes/sec + upload_speed: int # bytes/sec + seeders: int = 0 + peers: int = 0 + eta: Optional[int] = None # seconds, None if unknown + save_path: Optional[str] = None + files: Optional[List[str]] = None + error: Optional[str] = None + + +@runtime_checkable +class TorrentClientAdapter(Protocol): + """Structural contract every torrent-client adapter implements.""" + + def is_configured(self) -> bool: + """True when the adapter has a URL and any credentials it + needs. Reads from config_manager — never raises on missing + config, just returns False so the orchestrator can dim the + torrent download source in the UI.""" + ... + + async def check_connection(self) -> bool: + """Probe the client over the network. Logs in if required.""" + ... + + async def add_torrent( + self, + url_or_magnet: str, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + """Hand the torrent client a HTTP/HTTPS URL pointing to a + ``.torrent`` file or a ``magnet:`` URI. Returns the torrent's + client-side identifier (info-hash for qBit / Deluge, numeric + id for Transmission) or ``None`` on failure.""" + ... + + async def add_torrent_file( + self, + file_bytes: bytes, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + """Upload a raw ``.torrent`` payload. Same return as + ``add_torrent``. Used when the indexer doesn't expose a + direct download URL and SoulSync had to fetch the file + itself first.""" + ... + + async def get_status(self, torrent_id: str) -> Optional[TorrentStatus]: + """Return live status for one torrent, or ``None`` if the + client doesn't know about it.""" + ... + + async def get_all(self) -> List[TorrentStatus]: + """Return live status for every torrent the client currently + tracks. Used by the global download list.""" + ... + + async def remove(self, torrent_id: str, delete_files: bool = False) -> bool: + """Remove the torrent from the client. ``delete_files=True`` + also deletes the downloaded data on disk.""" + ... + + async def pause(self, torrent_id: str) -> bool: ... + + async def resume(self, torrent_id: str) -> bool: ... diff --git a/core/torrent_clients/deluge.py b/core/torrent_clients/deluge.py new file mode 100644 index 00000000..7c7e6ae9 --- /dev/null +++ b/core/torrent_clients/deluge.py @@ -0,0 +1,307 @@ +"""Deluge 2.x JSON-RPC adapter. + +Auth model: POST ``/json`` with method ``auth.login`` returns a +``_session_id`` cookie. The session cookie + every subsequent JSON-RPC +call must include a monotonically-incrementing ``id`` field. ``params`` +is a list (positional), not an object. + +Reference: https://deluge.readthedocs.io/en/latest/reference/api.html +and https://deluge.readthedocs.io/en/latest/reference/webapi.html +""" + +from __future__ import annotations + +import asyncio +import base64 +import threading +from itertools import count +from typing import Any, List, Optional + +import requests as http_requests + +from config.settings import config_manager +from core.torrent_clients.base import TorrentStatus +from utils.logging_config import get_logger + +logger = get_logger("torrent.deluge") + + +# Deluge native state strings → adapter-uniform set. +_DELUGE_STATE_MAP = { + "Allocating": "queued", + "Checking": "queued", + "Downloading": "downloading", + "Seeding": "seeding", + "Paused": "paused", + "Queued": "queued", + "Error": "error", + "Moving": "queued", +} + + +def _map_state(deluge_state: str, progress: float) -> str: + mapped = _DELUGE_STATE_MAP.get(deluge_state or '', "error") + if mapped == "paused" and progress >= 1.0: + return "completed" + return mapped + + +class DelugeAdapter: + """Deluge 2.x WebUI JSON-RPC adapter.""" + + DEFAULT_TIMEOUT = 15 + + # Fields we ask ``core.get_torrents_status`` to return — explicit + # to keep payload size predictable across Deluge versions that + # add fields over time. + _STATUS_FIELDS = [ + 'hash', 'name', 'state', 'progress', 'total_size', + 'total_done', 'download_payload_rate', 'upload_payload_rate', + 'num_seeds', 'num_peers', 'eta', 'save_path', 'tracker_status', + ] + + def __init__(self) -> None: + self._session: Optional[http_requests.Session] = None + self._session_lock = threading.Lock() + self._id_counter = count(1) + self._load_config() + + def _load_config(self) -> None: + self._url = (config_manager.get('torrent_client.url', '') or '').rstrip('/') + # Deluge's WebUI auth uses a single password, not username+password. + # We accept whichever field the user filled in — keeps the UI uniform. + self._password = ( + config_manager.get('torrent_client.password', '') + or config_manager.get('torrent_client.username', '') + or '' + ) + self._category = config_manager.get('torrent_client.category', 'soulsync') or 'soulsync' + self._save_path = config_manager.get('torrent_client.save_path', '') or '' + with self._session_lock: + self._session = None + + def reload_settings(self) -> None: + self._load_config() + + def is_configured(self) -> bool: + # Deluge WebUI requires a password; without it auth.login fails + # outright. Refuse the configuration up front rather than letting + # every call 401. + return bool(self._url and self._password) + + async def check_connection(self) -> bool: + if not self.is_configured(): + return False + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._check_connection_sync) + + def _check_connection_sync(self) -> bool: + sess = self._ensure_session_sync() + if sess is None: + return False + # web.connected() returns True iff the WebUI's daemon link is up. + # If daemons aren't auto-connected, fall back to a cheap call + # that just confirms we're authenticated. + result = self._rpc_sync('web.connected', []) + if result is True: + return True + # Fall back to a generic auth probe. + return self._rpc_sync('auth.check_session', []) is not None + + def _ensure_session_sync(self) -> Optional[http_requests.Session]: + with self._session_lock: + if self._session is not None: + return self._session + sess = http_requests.Session() + self._session = sess + # Log in. auth.login takes the password as a single positional arg. + result = self._rpc_sync('auth.login', [self._password]) + if result is not True: + logger.error("Deluge auth.login returned %r", result) + with self._session_lock: + self._session = None + return None + with self._session_lock: + return self._session + + def _rpc_sync(self, method: str, params: list) -> Any: + if not self._url: + return None + with self._session_lock: + sess = self._session + if sess is None: + # Bootstrap a session container; auth.login will populate it. + with self._session_lock: + if self._session is None: + self._session = http_requests.Session() + sess = self._session + payload = { + 'id': next(self._id_counter), + 'method': method, + 'params': params, + } + try: + resp = sess.post( + f"{self._url}/json", json=payload, + headers={'Content-Type': 'application/json'}, + timeout=self.DEFAULT_TIMEOUT, + ) + if not resp.ok: + logger.warning("Deluge %s returned HTTP %s", method, resp.status_code) + return None + data = resp.json() + err = data.get('error') + if err: + # Code 1 = unknown method, 2 = bad params, etc. + # Code 'No Auth' surfaces as a string in some versions. + logger.warning("Deluge %s error: %r", method, err) + return None + return data.get('result') + except Exception as e: + logger.error("Deluge %s call failed: %s", method, e) + return None + + async def add_torrent( + self, + url_or_magnet: str, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._add_torrent_sync, url_or_magnet, category, save_path + ) + + def _add_torrent_sync( + self, + url_or_magnet: str, + category: str, + save_path: Optional[str], + ) -> Optional[str]: + if self._ensure_session_sync() is None: + return None + options: dict = {} + if save_path or self._save_path: + options['download_location'] = save_path or self._save_path + # Deluge distinguishes magnet URIs from HTTP .torrent URLs at + # the API layer — different method names. + if url_or_magnet.startswith('magnet:'): + method = 'core.add_torrent_magnet' + else: + method = 'core.add_torrent_url' + torrent_hash = self._rpc_sync(method, [url_or_magnet, options]) + if not torrent_hash: + return None + self._apply_label(str(torrent_hash), category or self._category) + return str(torrent_hash) + + async def add_torrent_file( + self, + file_bytes: bytes, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._add_torrent_file_sync, file_bytes, category, save_path + ) + + def _add_torrent_file_sync( + self, + file_bytes: bytes, + category: str, + save_path: Optional[str], + ) -> Optional[str]: + if self._ensure_session_sync() is None: + return None + options: dict = {} + if save_path or self._save_path: + options['download_location'] = save_path or self._save_path + encoded = base64.b64encode(file_bytes).decode('ascii') + torrent_hash = self._rpc_sync('core.add_torrent_file', ['soulsync.torrent', encoded, options]) + if not torrent_hash: + return None + self._apply_label(str(torrent_hash), category or self._category) + return str(torrent_hash) + + def _apply_label(self, torrent_hash: str, label: str) -> None: + """Best-effort label assignment. The Label plugin is optional + in Deluge — if it isn't installed the RPC call fails silently + and we don't propagate the error: the torrent is still added.""" + if not label: + return + # Ensure the label exists before assigning. + self._rpc_sync('label.add', [label]) + self._rpc_sync('label.set_torrent', [torrent_hash, label]) + + async def get_status(self, torrent_id: str) -> Optional[TorrentStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_status_sync, torrent_id) + + def _get_status_sync(self, torrent_id: str) -> Optional[TorrentStatus]: + result = self._rpc_sync('core.get_torrent_status', [torrent_id, self._STATUS_FIELDS]) + if not isinstance(result, dict) or not result: + return None + # core.get_torrent_status doesn't echo back the hash — patch it in. + result.setdefault('hash', torrent_id) + return self._parse_status(result) + + async def get_all(self) -> List[TorrentStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_all_sync) + + def _get_all_sync(self) -> List[TorrentStatus]: + result = self._rpc_sync('core.get_torrents_status', [{}, self._STATUS_FIELDS]) + if not isinstance(result, dict): + return [] + out = [] + for hash_id, item in result.items(): + if not isinstance(item, dict): + continue + item.setdefault('hash', hash_id) + out.append(self._parse_status(item)) + return out + + def _parse_status(self, item: dict) -> TorrentStatus: + progress = float(item.get('progress') or 0.0) + # Deluge expresses progress as 0-100; normalize to 0-1. + if progress > 1.0: + progress = progress / 100.0 + return TorrentStatus( + id=str(item.get('hash') or ''), + name=item.get('name') or '', + state=_map_state(item.get('state') or '', progress), + progress=progress, + size=int(item.get('total_size') or 0), + downloaded=int(item.get('total_done') or 0), + download_speed=int(item.get('download_payload_rate') or 0), + upload_speed=int(item.get('upload_payload_rate') or 0), + seeders=int(item.get('num_seeds') or 0), + peers=int(item.get('num_peers') or 0), + eta=int(item['eta']) if isinstance(item.get('eta'), (int, float)) and item.get('eta', 0) > 0 else None, + save_path=item.get('save_path'), + error=item.get('tracker_status') if 'Error' in (item.get('state') or '') else None, + ) + + async def remove(self, torrent_id: str, delete_files: bool = False) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._remove_sync, torrent_id, delete_files) + + def _remove_sync(self, torrent_id: str, delete_files: bool) -> bool: + result = self._rpc_sync('core.remove_torrent', [torrent_id, bool(delete_files)]) + return result is True + + async def pause(self, torrent_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._pause_sync, torrent_id) + + def _pause_sync(self, torrent_id: str) -> bool: + # Deluge 2.x core.pause_torrent takes a list of hashes. + return self._rpc_sync('core.pause_torrent', [[torrent_id]]) is not None + + async def resume(self, torrent_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._resume_sync, torrent_id) + + def _resume_sync(self, torrent_id: str) -> bool: + return self._rpc_sync('core.resume_torrent', [[torrent_id]]) is not None diff --git a/core/torrent_clients/qbittorrent.py b/core/torrent_clients/qbittorrent.py new file mode 100644 index 00000000..579c8ec8 --- /dev/null +++ b/core/torrent_clients/qbittorrent.py @@ -0,0 +1,325 @@ +"""qBittorrent WebUI v2 adapter. + +Auth model: POST ``/api/v2/auth/login`` with form-encoded +``username`` + ``password`` returns a ``SID`` cookie that's required +on every subsequent call. The cookie lives on a ``requests.Session`` +maintained by this adapter; we lazily re-login on 403 in case the +server expired the session. + +Reference: https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1) +""" + +from __future__ import annotations + +import asyncio +import threading +from typing import List, Optional + +import requests as http_requests + +from config.settings import config_manager +from core.torrent_clients.base import TorrentStatus +from utils.logging_config import get_logger + +logger = get_logger("torrent.qbittorrent") + + +# qBittorrent's native state strings. Mapped onto the adapter-uniform +# set in ``_map_state``. See https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1)#torrent-list +_QBIT_STATE_MAP = { + "allocating": "queued", + "checkingDL": "queued", + "checkingUP": "seeding", + "checkingResumeData": "queued", + "downloading": "downloading", + "error": "error", + "forcedDL": "downloading", + "forcedUP": "seeding", + "metaDL": "downloading", + "missingFiles": "error", + "moving": "queued", + "pausedDL": "paused", + "pausedUP": "completed", + "queuedDL": "queued", + "queuedUP": "queued", + "stalledDL": "stalled", + "stalledUP": "seeding", + "uploading": "seeding", + "unknown": "error", +} + + +def _map_state(qbit_state: str) -> str: + return _QBIT_STATE_MAP.get(qbit_state, "error") + + +class QBittorrentAdapter: + """qBittorrent WebUI v2 adapter.""" + + DEFAULT_TIMEOUT = 15 + + def __init__(self) -> None: + self._session: Optional[http_requests.Session] = None + self._session_lock = threading.Lock() + self._load_config() + + def _load_config(self) -> None: + self._url = (config_manager.get('torrent_client.url', '') or '').rstrip('/') + self._username = config_manager.get('torrent_client.username', '') or '' + self._password = config_manager.get('torrent_client.password', '') or '' + self._category = config_manager.get('torrent_client.category', 'soulsync') or 'soulsync' + self._save_path = config_manager.get('torrent_client.save_path', '') or '' + # Drop any existing session — credentials may have changed. + with self._session_lock: + self._session = None + + def reload_settings(self) -> None: + self._load_config() + + def is_configured(self) -> bool: + # qBittorrent allows no-auth setups (LAN), so credentials are + # optional — URL is the only hard requirement. + return bool(self._url) + + async def check_connection(self) -> bool: + if not self.is_configured(): + return False + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._check_connection_sync) + + def _check_connection_sync(self) -> bool: + try: + sess = self._ensure_session_sync() + if sess is None: + return False + resp = sess.get(f"{self._url}/api/v2/app/version", timeout=self.DEFAULT_TIMEOUT) + return resp.ok + except Exception as e: + logger.error("qBittorrent connection probe failed: %s", e) + return False + + def _ensure_session_sync(self) -> Optional[http_requests.Session]: + with self._session_lock: + if self._session is not None: + return self._session + sess = http_requests.Session() + # No-auth setup — skip login. + if not self._username and not self._password: + self._session = sess + return sess + try: + resp = sess.post( + f"{self._url}/api/v2/auth/login", + data={'username': self._username, 'password': self._password}, + # qBittorrent rejects login attempts that arrive without a + # Referer matching its configured host (CSRF guard). Sending + # the WebUI's own URL satisfies the check. + headers={'Referer': self._url}, + timeout=self.DEFAULT_TIMEOUT, + ) + if not resp.ok or resp.text.strip() != 'Ok.': + logger.error("qBittorrent login failed: HTTP %s body=%r", resp.status_code, resp.text[:200]) + return None + self._session = sess + return sess + except Exception as e: + logger.error("qBittorrent login error: %s", e) + return None + + def _call(self, method: str, path: str, **kwargs) -> Optional[http_requests.Response]: + sess = self._ensure_session_sync() + if sess is None: + return None + try: + kwargs.setdefault('timeout', self.DEFAULT_TIMEOUT) + kwargs.setdefault('headers', {}).setdefault('Referer', self._url) + resp = sess.request(method, f"{self._url}{path}", **kwargs) + # Session expired — try one re-login and retry. + if resp.status_code == 403: + with self._session_lock: + self._session = None + sess = self._ensure_session_sync() + if sess is None: + return None + resp = sess.request(method, f"{self._url}{path}", **kwargs) + return resp + except Exception as e: + logger.error("qBittorrent %s %s failed: %s", method, path, e) + return None + + async def add_torrent( + self, + url_or_magnet: str, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._add_torrent_sync, url_or_magnet, category, save_path + ) + + def _add_torrent_sync( + self, + url_or_magnet: str, + category: str, + save_path: Optional[str], + ) -> Optional[str]: + cat = category or self._category + # Snapshot the current set of torrent hashes BEFORE adding — + # qBittorrent's /add endpoint returns 200 "Ok." regardless of + # whether the URL was actually accepted or registered, and + # category-filtered lookups race the add (qBit hasn't + # categorised the new torrent yet on the first poll). Diffing + # before / after is the only reliable way to recover the hash. + before = self._all_hashes() + if before is None: + return None + data = {'urls': url_or_magnet, 'category': cat} + if save_path or self._save_path: + data['savepath'] = save_path or self._save_path + resp = self._call('POST', '/api/v2/torrents/add', data=data) + if not resp or not resp.ok: + logger.warning("qBittorrent /torrents/add returned HTTP %s body=%r", + resp.status_code if resp else 'no-response', + (resp.text[:200] if resp else '')) + return None + if resp.text and resp.text.strip() and resp.text.strip() != 'Ok.': + logger.warning("qBittorrent /torrents/add unexpected body: %r", resp.text[:200]) + return None + new_hash = self._poll_for_new_hash(before) + if not new_hash: + logger.error("qBittorrent accepted the request but no new torrent appeared — " + "URL may have been rejected (bad magnet, unreachable HTTPS, " + "duplicate hash, etc.)") + return new_hash + + def _all_hashes(self) -> Optional[set]: + """Return the set of every torrent hash qBit currently tracks, + or None on lookup failure.""" + resp = self._call('GET', '/api/v2/torrents/info') + if not resp or not resp.ok: + return None + try: + return {item.get('hash') for item in resp.json() if item.get('hash')} + except Exception as e: + logger.error("qBittorrent /torrents/info parse failed: %s", e) + return None + + def _poll_for_new_hash(self, before: set) -> Optional[str]: + """Poll up to ~5s for a new torrent to appear (qBit takes a + moment to fetch the .torrent file from the URL and register + it). Returns the new hash, or None if nothing showed up.""" + import time as _time + for _ in range(10): + _time.sleep(0.5) + current = self._all_hashes() + if current is None: + continue + new = current - before + if new: + return next(iter(new)) + return None + + async def add_torrent_file( + self, + file_bytes: bytes, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._add_torrent_file_sync, file_bytes, category, save_path + ) + + def _add_torrent_file_sync( + self, + file_bytes: bytes, + category: str, + save_path: Optional[str], + ) -> Optional[str]: + cat = category or self._category + before = self._all_hashes() + if before is None: + return None + data = {'category': cat} + if save_path or self._save_path: + data['savepath'] = save_path or self._save_path + files = {'torrents': ('soulsync.torrent', file_bytes, 'application/x-bittorrent')} + resp = self._call('POST', '/api/v2/torrents/add', data=data, files=files) + if not resp or not resp.ok: + return None + return self._poll_for_new_hash(before) + + async def get_status(self, torrent_id: str) -> Optional[TorrentStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_status_sync, torrent_id) + + def _get_status_sync(self, torrent_id: str) -> Optional[TorrentStatus]: + resp = self._call('GET', '/api/v2/torrents/info', params={'hashes': torrent_id}) + if not resp or not resp.ok: + return None + try: + items = resp.json() + except Exception as e: + logger.error("qBittorrent get_status parse failed: %s", e) + return None + if not items: + return None + return self._parse_status(items[0]) + + async def get_all(self) -> List[TorrentStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_all_sync) + + def _get_all_sync(self) -> List[TorrentStatus]: + resp = self._call('GET', '/api/v2/torrents/info') + if not resp or not resp.ok: + return [] + try: + return [self._parse_status(item) for item in resp.json()] + except Exception as e: + logger.error("qBittorrent get_all parse failed: %s", e) + return [] + + def _parse_status(self, item: dict) -> TorrentStatus: + return TorrentStatus( + id=str(item.get('hash') or ''), + name=item.get('name') or '', + state=_map_state(item.get('state') or 'unknown'), + progress=float(item.get('progress') or 0.0), + size=int(item.get('size') or 0), + downloaded=int(item.get('downloaded') or 0), + download_speed=int(item.get('dlspeed') or 0), + upload_speed=int(item.get('upspeed') or 0), + seeders=int(item.get('num_seeds') or 0), + peers=int(item.get('num_leechs') or 0), + eta=item.get('eta') if isinstance(item.get('eta'), int) and item.get('eta', 0) > 0 else None, + save_path=item.get('save_path'), + ) + + async def remove(self, torrent_id: str, delete_files: bool = False) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._remove_sync, torrent_id, delete_files) + + def _remove_sync(self, torrent_id: str, delete_files: bool) -> bool: + resp = self._call('POST', '/api/v2/torrents/delete', data={ + 'hashes': torrent_id, + 'deleteFiles': 'true' if delete_files else 'false', + }) + return bool(resp and resp.ok) + + async def pause(self, torrent_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._pause_sync, torrent_id) + + def _pause_sync(self, torrent_id: str) -> bool: + resp = self._call('POST', '/api/v2/torrents/pause', data={'hashes': torrent_id}) + return bool(resp and resp.ok) + + async def resume(self, torrent_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._resume_sync, torrent_id) + + def _resume_sync(self, torrent_id: str) -> bool: + resp = self._call('POST', '/api/v2/torrents/resume', data={'hashes': torrent_id}) + return bool(resp and resp.ok) diff --git a/core/torrent_clients/transmission.py b/core/torrent_clients/transmission.py new file mode 100644 index 00000000..266bda79 --- /dev/null +++ b/core/torrent_clients/transmission.py @@ -0,0 +1,266 @@ +"""Transmission RPC adapter. + +Auth model: Transmission uses HTTP Basic auth + an +``X-Transmission-Session-Id`` header. The session ID rotates and is +returned on every 409 response — the adapter caches the latest value +and replays the original request transparently. + +Reference: https://github.com/transmission/transmission/blob/main/docs/rpc-spec.md +""" + +from __future__ import annotations + +import asyncio +import base64 +import threading +from typing import List, Optional + +import requests as http_requests + +from config.settings import config_manager +from core.torrent_clients.base import TorrentStatus +from utils.logging_config import get_logger + +logger = get_logger("torrent.transmission") + + +# Transmission RPC status codes. Defined as numeric constants in the +# RPC spec — converted to the adapter-uniform string set here. +_TRANSMISSION_STATUS = { + 0: "paused", + 1: "queued", # queued to check files + 2: "queued", # checking files + 3: "queued", # queued to download + 4: "downloading", + 5: "queued", # queued to seed + 6: "seeding", +} + + +def _map_state(status_code: int, percent_done: float) -> str: + base = _TRANSMISSION_STATUS.get(status_code, "error") + # Transmission reports 'paused' (0) for both never-started and + # fully-downloaded-but-not-seeding. Differentiate by progress. + if status_code == 0 and percent_done >= 1.0: + return "completed" + return base + + +class TransmissionAdapter: + """Transmission RPC adapter (transmission-rpc protocol v17+).""" + + DEFAULT_TIMEOUT = 15 + + def __init__(self) -> None: + self._session_id: Optional[str] = None + self._session_id_lock = threading.Lock() + self._load_config() + + def _load_config(self) -> None: + url = (config_manager.get('torrent_client.url', '') or '').rstrip('/') + # Transmission's RPC endpoint is always /transmission/rpc — if the + # user pasted a bare host URL, append it. If they pasted the full + # /transmission/rpc URL, leave it. + if url and not url.endswith('/transmission/rpc'): + if '/transmission' not in url: + url = f"{url}/transmission/rpc" + self._url = url + self._username = config_manager.get('torrent_client.username', '') or '' + self._password = config_manager.get('torrent_client.password', '') or '' + self._category = config_manager.get('torrent_client.category', 'soulsync') or 'soulsync' + self._save_path = config_manager.get('torrent_client.save_path', '') or '' + with self._session_id_lock: + self._session_id = None + + def reload_settings(self) -> None: + self._load_config() + + def is_configured(self) -> bool: + return bool(self._url) + + async def check_connection(self) -> bool: + if not self.is_configured(): + return False + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._check_connection_sync) + + def _check_connection_sync(self) -> bool: + resp = self._rpc('session-get', {}) + return resp is not None + + def _rpc(self, method: str, arguments: dict) -> Optional[dict]: + """One Transmission RPC round-trip. Handles the 409 + session-id renegotiation transparently — Transmission rejects + the first call with HTTP 409 and a fresh + ``X-Transmission-Session-Id`` header, which subsequent calls + must echo back.""" + if not self._url: + return None + auth = (self._username, self._password) if self._username else None + payload = {'method': method, 'arguments': arguments} + for _attempt in range(2): + try: + with self._session_id_lock: + sid = self._session_id + headers = {'Content-Type': 'application/json'} + if sid: + headers['X-Transmission-Session-Id'] = sid + resp = http_requests.post( + self._url, json=payload, headers=headers, auth=auth, + timeout=self.DEFAULT_TIMEOUT, + ) + if resp.status_code == 409: + # Pick up the new session id and retry. + new_sid = resp.headers.get('X-Transmission-Session-Id') + if not new_sid: + logger.error("Transmission 409 with no X-Transmission-Session-Id header") + return None + with self._session_id_lock: + self._session_id = new_sid + continue + if not resp.ok: + logger.warning("Transmission RPC %s returned HTTP %s", method, resp.status_code) + return None + data = resp.json() + if data.get('result') != 'success': + logger.warning("Transmission RPC %s result=%s", method, data.get('result')) + return None + return data.get('arguments', {}) + except Exception as e: + logger.error("Transmission RPC %s failed: %s", method, e) + return None + return None + + async def add_torrent( + self, + url_or_magnet: str, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._add_torrent_sync, url_or_magnet, category, save_path + ) + + def _add_torrent_sync( + self, + url_or_magnet: str, + category: str, + save_path: Optional[str], + ) -> Optional[str]: + args: dict = {'filename': url_or_magnet, 'labels': [category or self._category]} + if save_path or self._save_path: + args['download-dir'] = save_path or self._save_path + return self._add_torrent_finish(args) + + async def add_torrent_file( + self, + file_bytes: bytes, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._add_torrent_file_sync, file_bytes, category, save_path + ) + + def _add_torrent_file_sync( + self, + file_bytes: bytes, + category: str, + save_path: Optional[str], + ) -> Optional[str]: + args: dict = { + 'metainfo': base64.b64encode(file_bytes).decode('ascii'), + 'labels': [category or self._category], + } + if save_path or self._save_path: + args['download-dir'] = save_path or self._save_path + return self._add_torrent_finish(args) + + def _add_torrent_finish(self, args: dict) -> Optional[str]: + data = self._rpc('torrent-add', args) + if not data: + return None + # torrent-add returns either ``torrent-added`` (new torrent) or + # ``torrent-duplicate`` (already exists). Both carry the hash. + torrent = data.get('torrent-added') or data.get('torrent-duplicate') + if not torrent: + return None + return str(torrent.get('hashString') or '') or None + + async def get_status(self, torrent_id: str) -> Optional[TorrentStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_status_sync, torrent_id) + + def _get_status_sync(self, torrent_id: str) -> Optional[TorrentStatus]: + data = self._rpc('torrent-get', { + 'ids': [torrent_id], + 'fields': self._STATUS_FIELDS, + }) + if not data or not data.get('torrents'): + return None + return self._parse_status(data['torrents'][0]) + + async def get_all(self) -> List[TorrentStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_all_sync) + + def _get_all_sync(self) -> List[TorrentStatus]: + data = self._rpc('torrent-get', {'fields': self._STATUS_FIELDS}) + if not data: + return [] + return [self._parse_status(t) for t in data.get('torrents', [])] + + _STATUS_FIELDS = [ + 'hashString', 'name', 'status', 'percentDone', 'totalSize', + 'downloadedEver', 'rateDownload', 'rateUpload', 'peersSendingToUs', + 'peersGettingFromUs', 'eta', 'downloadDir', 'errorString', + ] + + def _parse_status(self, item: dict) -> TorrentStatus: + progress = float(item.get('percentDone') or 0.0) + status_code = int(item.get('status') or 0) + eta_raw = item.get('eta') + # Transmission returns -1 / -2 for "unknown" — surface as None. + eta = eta_raw if isinstance(eta_raw, int) and eta_raw > 0 else None + return TorrentStatus( + id=str(item.get('hashString') or ''), + name=item.get('name') or '', + state=_map_state(status_code, progress), + progress=progress, + size=int(item.get('totalSize') or 0), + downloaded=int(item.get('downloadedEver') or 0), + download_speed=int(item.get('rateDownload') or 0), + upload_speed=int(item.get('rateUpload') or 0), + seeders=int(item.get('peersSendingToUs') or 0), + peers=int(item.get('peersGettingFromUs') or 0), + eta=eta, + save_path=item.get('downloadDir'), + error=item.get('errorString') or None, + ) + + async def remove(self, torrent_id: str, delete_files: bool = False) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._remove_sync, torrent_id, delete_files) + + def _remove_sync(self, torrent_id: str, delete_files: bool) -> bool: + data = self._rpc('torrent-remove', { + 'ids': [torrent_id], + 'delete-local-data': bool(delete_files), + }) + return data is not None + + async def pause(self, torrent_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._pause_sync, torrent_id) + + def _pause_sync(self, torrent_id: str) -> bool: + return self._rpc('torrent-stop', {'ids': [torrent_id]}) is not None + + async def resume(self, torrent_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._resume_sync, torrent_id) + + def _resume_sync(self, torrent_id: str) -> bool: + return self._rpc('torrent-start', {'ids': [torrent_id]}) is not None diff --git a/core/usenet_clients/__init__.py b/core/usenet_clients/__init__.py new file mode 100644 index 00000000..640b3ef3 --- /dev/null +++ b/core/usenet_clients/__init__.py @@ -0,0 +1,49 @@ +"""Usenet client adapters. + +Each adapter wraps one Usenet downloader (SABnzbd, NZBGet) behind +the ``UsenetClientAdapter`` Protocol so the rest of SoulSync can +talk to whichever client the user picked through one uniform +surface. + +The active adapter is selected at runtime by the +``usenet_client.type`` config key. See ``get_active_adapter()`` +for the factory. +""" + +from __future__ import annotations + +from typing import Optional + +from config.settings import config_manager + +from core.usenet_clients.base import UsenetClientAdapter, UsenetStatus +from core.usenet_clients.nzbget import NZBGetAdapter +from core.usenet_clients.sabnzbd import SABnzbdAdapter + +__all__ = [ + "UsenetClientAdapter", + "UsenetStatus", + "SABnzbdAdapter", + "NZBGetAdapter", + "get_active_adapter", + "adapter_for_type", +] + + +def adapter_for_type(client_type: str) -> Optional[UsenetClientAdapter]: + """Build a fresh adapter instance for the given client type string. + ``None`` for unknown types.""" + if client_type == "sabnzbd": + return SABnzbdAdapter() + if client_type == "nzbget": + return NZBGetAdapter() + return None + + +def get_active_adapter() -> Optional[UsenetClientAdapter]: + """Return an adapter for whichever usenet client the user has + selected in Settings. Reads ``usenet_client.type`` each call.""" + client_type = (config_manager.get('usenet_client.type', '') or '').strip().lower() + if not client_type: + return None + return adapter_for_type(client_type) diff --git a/core/usenet_clients/base.py b/core/usenet_clients/base.py new file mode 100644 index 00000000..f77b3d78 --- /dev/null +++ b/core/usenet_clients/base.py @@ -0,0 +1,74 @@ +"""Usenet client adapter contract. + +``UsenetClientAdapter`` mirrors ``TorrentClientAdapter`` in shape so +the download plugin layer can reuse the same dispatch pattern. +Differences from the torrent side: + +- No magnet URI equivalent — usenet jobs are always ``.nzb`` files + or URLs that resolve to one. +- No seed/peer counts — usenet is a download-only protocol. +- Status values reflect usenet semantics: ``downloading`` / + ``extracting`` / ``verifying`` / ``repairing`` / ``completed`` / + ``failed`` / ``paused``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Protocol, runtime_checkable + + +@dataclass +class UsenetStatus: + """Adapter-uniform view of one usenet job. + + Field semantics: + - ``state`` is one of: ``queued`` | ``downloading`` | ``extracting`` + | ``verifying`` | ``repairing`` | ``completed`` | ``failed`` | + ``paused``. Each adapter maps its native names to this set. + - ``progress`` is 0.0–1.0 across the entire job (download + par2 + + unpack), so a job stalled at the verify step still shows < 1.0. + """ + + id: str # SAB nzo_id / NZBGet NZBID + name: str + state: str + progress: float + size: int # total size in bytes + downloaded: int # bytes downloaded so far + download_speed: int # bytes/sec + eta: Optional[int] = None # seconds, None if unknown + save_path: Optional[str] = None + category: Optional[str] = None + files: Optional[List[str]] = None + error: Optional[str] = None + + +@runtime_checkable +class UsenetClientAdapter(Protocol): + """Structural contract every usenet-client adapter implements.""" + + def is_configured(self) -> bool: ... + + async def check_connection(self) -> bool: ... + + async def add_nzb( + self, + url_or_bytes, + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + """Hand the usenet client either a ``.nzb`` HTTP URL (``str``) + or the raw payload (``bytes``). Returns the client-side job id + on success, ``None`` on failure.""" + ... + + async def get_status(self, job_id: str) -> Optional[UsenetStatus]: ... + + async def get_all(self) -> List[UsenetStatus]: ... + + async def remove(self, job_id: str, delete_files: bool = False) -> bool: ... + + async def pause(self, job_id: str) -> bool: ... + + async def resume(self, job_id: str) -> bool: ... diff --git a/core/usenet_clients/nzbget.py b/core/usenet_clients/nzbget.py new file mode 100644 index 00000000..d58074ff --- /dev/null +++ b/core/usenet_clients/nzbget.py @@ -0,0 +1,277 @@ +"""NZBGet adapter. + +Auth model: HTTP Basic auth on the JSON-RPC endpoint ``/jsonrpc``. +Every method takes positional ``params``. Identical pattern to +Deluge but with different method names. + +Reference: https://nzbget.com/documentation/api/ +""" + +from __future__ import annotations + +import asyncio +import base64 +from itertools import count +from typing import Any, List, Optional, Union + +import requests as http_requests + +from config.settings import config_manager +from core.usenet_clients.base import UsenetStatus +from utils.logging_config import get_logger + +logger = get_logger("usenet.nzbget") + + +# NZBGet's ``Status`` field on ListGroups → adapter-uniform set. +# NZBGet states (group): QUEUED, PAUSED, DOWNLOADING, FETCHING, PP_QUEUED, +# LOADING_PARS, VERIFYING_SOURCES, REPAIRING, VERIFYING_REPAIRED, RENAMING, +# UNPACKING, MOVING, EXECUTING_SCRIPT, PP_FINISHED. +_NZBGET_STATE_MAP = { + "QUEUED": "queued", + "PAUSED": "paused", + "DOWNLOADING": "downloading", + "FETCHING": "downloading", + "PP_QUEUED": "queued", + "LOADING_PARS": "verifying", + "VERIFYING_SOURCES": "verifying", + "REPAIRING": "repairing", + "VERIFYING_REPAIRED": "verifying", + "RENAMING": "extracting", + "UNPACKING": "extracting", + "MOVING": "extracting", + "EXECUTING_SCRIPT": "extracting", + "PP_FINISHED": "completed", +} + + +def _map_state(nzbget_state: str) -> str: + return _NZBGET_STATE_MAP.get(nzbget_state or '', "error") + + +class NZBGetAdapter: + """NZBGet JSON-RPC adapter.""" + + DEFAULT_TIMEOUT = 15 + + def __init__(self) -> None: + self._id_counter = count(1) + self._load_config() + + def _load_config(self) -> None: + self._url = (config_manager.get('usenet_client.url', '') or '').rstrip('/') + self._username = config_manager.get('usenet_client.username', '') or '' + self._password = config_manager.get('usenet_client.password', '') or '' + self._category = config_manager.get('usenet_client.category', 'soulsync') or 'soulsync' + + def reload_settings(self) -> None: + self._load_config() + + def is_configured(self) -> bool: + return bool(self._url and self._username and self._password) + + async def check_connection(self) -> bool: + if not self.is_configured(): + return False + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._check_connection_sync) + + def _check_connection_sync(self) -> bool: + return self._rpc_sync('version', []) is not None + + def _rpc_sync(self, method: str, params: list) -> Any: + if not self._url: + return None + try: + resp = http_requests.post( + f"{self._url}/jsonrpc", + json={'method': method, 'params': params, 'id': next(self._id_counter)}, + auth=(self._username, self._password) if self._username else None, + headers={'Content-Type': 'application/json'}, + timeout=self.DEFAULT_TIMEOUT, + ) + if not resp.ok: + logger.warning("NZBGet %s returned HTTP %s", method, resp.status_code) + return None + data = resp.json() + if data.get('error'): + logger.warning("NZBGet %s error: %r", method, data.get('error')) + return None + return data.get('result') + except http_requests.exceptions.RequestException as e: + logger.error("NZBGet %s call failed: %s", method, e) + return None + except ValueError as e: + logger.error("NZBGet %s response not JSON: %s", method, e) + return None + + async def add_nzb( + self, + url_or_bytes: Union[str, bytes], + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._add_nzb_sync, url_or_bytes, category, save_path + ) + + def _add_nzb_sync( + self, + url_or_bytes: Union[str, bytes], + category: str, + save_path: Optional[str], + ) -> Optional[str]: + cat = category or self._category + # NZBGet's ``append`` takes: NZBFilename, Content, Category, + # Priority, AddToTop, AddPaused, DupeKey, DupeScore, DupeMode, + # PPParameters. We pass the minimum required for an unpause-on-add. + # Content is either base64 of the raw .nzb or a URL — NZBGet + # auto-detects which based on whether it looks like a URL. + if isinstance(url_or_bytes, bytes): + content = base64.b64encode(url_or_bytes).decode('ascii') + nzb_filename = 'soulsync.nzb' + else: + content = url_or_bytes + nzb_filename = '' + params = [ + nzb_filename, # NZBFilename + content, # Content (URL or base64 NZB) + cat, # Category + 0, # Priority + False, # AddToTop + False, # AddPaused + '', # DupeKey + 0, # DupeScore + 'SCORE', # DupeMode + [], # PPParameters + ] + result = self._rpc_sync('append', params) + if isinstance(result, int) and result > 0: + return str(result) + return None + + async def get_status(self, job_id: str) -> Optional[UsenetStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_status_sync, job_id) + + def _get_status_sync(self, job_id: str) -> Optional[UsenetStatus]: + for status in self._get_all_sync(): + if status.id == job_id: + return status + return None + + async def get_all(self) -> List[UsenetStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_all_sync) + + def _get_all_sync(self) -> List[UsenetStatus]: + out: List[UsenetStatus] = [] + groups = self._rpc_sync('listgroups', [0]) + if isinstance(groups, list): + for group in groups: + out.append(self._parse_group(group)) + history = self._rpc_sync('history', [False]) + if isinstance(history, list): + for entry in history: + out.append(self._parse_history(entry)) + return out + + def _parse_group(self, group: dict) -> UsenetStatus: + # NZBGet reports sizes split into ``FileSizeLo`` (low 32 bits) + + # ``FileSizeHi`` (high 32 bits) for compat with old clients — + # ``FileSizeMB`` is the human-friendly aggregate. + size_mb = self._mb_value(group, 'FileSize') + remaining_mb = self._mb_value(group, 'RemainingSize') + size_bytes = int(size_mb * 1024 * 1024) if size_mb else 0 + downloaded_bytes = int((size_mb - remaining_mb) * 1024 * 1024) if size_mb and remaining_mb is not None else 0 + progress = 0.0 + if size_bytes > 0: + progress = max(0.0, min(downloaded_bytes / size_bytes, 1.0)) + # NZBGet's per-group ``DownloadRate`` field is in bytes/sec. + speed = int(group.get('DownloadRate') or 0) + return UsenetStatus( + id=str(group.get('NZBID') or ''), + name=group.get('NZBName') or '', + state=_map_state(group.get('Status') or ''), + progress=progress, + size=size_bytes, + downloaded=downloaded_bytes, + download_speed=speed, + save_path=group.get('DestDir'), + category=group.get('Category'), + ) + + def _parse_history(self, entry: dict) -> UsenetStatus: + # History entries have ``Status`` like ``SUCCESS/HEALTH``, + # ``SUCCESS/UNPACK``, ``FAILURE/PAR``, etc. + status_field = entry.get('Status') or '' + is_failed = status_field.startswith('FAILURE') + size_mb = self._mb_value(entry, 'FileSize') + size_bytes = int(size_mb * 1024 * 1024) if size_mb else 0 + return UsenetStatus( + id=str(entry.get('NZBID') or ''), + name=entry.get('Name') or entry.get('NZBName') or '', + state='failed' if is_failed else 'completed', + progress=0.0 if is_failed else 1.0, + size=size_bytes, + downloaded=size_bytes if not is_failed else 0, + download_speed=0, + save_path=entry.get('DestDir'), + category=entry.get('Category'), + error=status_field if is_failed else None, + ) + + @staticmethod + def _mb_value(entry: dict, prefix: str) -> Optional[float]: + """Read an NZBGet size field. Prefers the high+low 32-bit split + when available (most accurate); falls back to the ``MB`` + aggregate for older NZBGet versions.""" + lo = entry.get(f'{prefix}Lo') + hi = entry.get(f'{prefix}Hi') + if isinstance(lo, int) and isinstance(hi, int): + total_bytes = (hi << 32) | lo + return total_bytes / (1024 * 1024) + mb = entry.get(f'{prefix}MB') + if isinstance(mb, (int, float)): + return float(mb) + return None + + async def remove(self, job_id: str, delete_files: bool = False) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._remove_sync, job_id, delete_files) + + def _remove_sync(self, job_id: str, delete_files: bool) -> bool: + # editqueue commands take a list of NZBIDs. ``GroupFinalDelete`` + # both removes and deletes downloaded data; ``GroupDelete`` just + # removes the queue entry. + try: + id_int = int(job_id) + except (TypeError, ValueError): + return False + command = 'GroupFinalDelete' if delete_files else 'GroupDelete' + # editqueue(Command, Offset, EditText, IDs) + result = self._rpc_sync('editqueue', [command, 0, '', [id_int]]) + return bool(result) + + async def pause(self, job_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._pause_sync, job_id) + + def _pause_sync(self, job_id: str) -> bool: + try: + id_int = int(job_id) + except (TypeError, ValueError): + return False + return bool(self._rpc_sync('editqueue', ['GroupPause', 0, '', [id_int]])) + + async def resume(self, job_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._resume_sync, job_id) + + def _resume_sync(self, job_id: str) -> bool: + try: + id_int = int(job_id) + except (TypeError, ValueError): + return False + return bool(self._rpc_sync('editqueue', ['GroupResume', 0, '', [id_int]])) diff --git a/core/usenet_clients/sabnzbd.py b/core/usenet_clients/sabnzbd.py new file mode 100644 index 00000000..0cf3428c --- /dev/null +++ b/core/usenet_clients/sabnzbd.py @@ -0,0 +1,284 @@ +"""SABnzbd adapter. + +Auth model: a single API key passed as ``?apikey=...`` on every +request. No login flow. Every endpoint is the same path ``/api`` with +a ``mode=`` query param. + +Reference: https://sabnzbd.org/wiki/configuration/4.3/api +""" + +from __future__ import annotations + +import asyncio +from typing import List, Optional, Union + +import requests as http_requests + +from config.settings import config_manager +from core.usenet_clients.base import UsenetStatus +from utils.logging_config import get_logger + +logger = get_logger("usenet.sabnzbd") + + +# SAB queue states + history states → adapter-uniform set. +# Queue: Idle, Paused, Downloading, Grabbing, Queued, Checking, +# QuickCheck, Verifying, Repairing, Fetching, Extracting, Moving, +# Running, Completed, Failed. +_SAB_QUEUE_STATE_MAP = { + "idle": "queued", + "queued": "queued", + "grabbing": "queued", + "fetching": "downloading", + "downloading": "downloading", + "paused": "paused", + "checking": "verifying", + "quickcheck": "verifying", + "verifying": "verifying", + "repairing": "repairing", + "extracting": "extracting", + "moving": "extracting", + "running": "extracting", + "completed": "completed", + "failed": "failed", +} + + +def _map_state(sab_state: str) -> str: + return _SAB_QUEUE_STATE_MAP.get((sab_state or "").lower(), "error") + + +class SABnzbdAdapter: + """SABnzbd REST API adapter (v2+).""" + + DEFAULT_TIMEOUT = 15 + + def __init__(self) -> None: + self._load_config() + + def _load_config(self) -> None: + self._url = (config_manager.get('usenet_client.url', '') or '').rstrip('/') + self._api_key = config_manager.get('usenet_client.api_key', '') or '' + self._category = config_manager.get('usenet_client.category', 'soulsync') or 'soulsync' + + def reload_settings(self) -> None: + self._load_config() + + def is_configured(self) -> bool: + return bool(self._url and self._api_key) + + async def check_connection(self) -> bool: + if not self.is_configured(): + return False + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._check_connection_sync) + + def _check_connection_sync(self) -> bool: + # ``mode=version`` is the cheapest authenticated probe SAB exposes. + data = self._call_sync('version') + return bool(data and data.get('version')) + + def _call_sync(self, mode: str, **extra) -> Optional[dict]: + if not self.is_configured(): + return None + params = { + 'mode': mode, + 'output': 'json', + 'apikey': self._api_key, + } + params.update(extra) + try: + resp = http_requests.get(f"{self._url}/api", params=params, timeout=self.DEFAULT_TIMEOUT) + if not resp.ok: + logger.warning("SABnzbd mode=%s returned HTTP %s", mode, resp.status_code) + return None + return resp.json() + except http_requests.exceptions.RequestException as e: + logger.error("SABnzbd mode=%s request failed: %s", mode, e) + return None + except ValueError as e: + logger.error("SABnzbd mode=%s response was not JSON: %s", mode, e) + return None + + def _post_sync(self, mode: str, files=None, **extra) -> Optional[dict]: + if not self.is_configured(): + return None + params = { + 'mode': mode, + 'output': 'json', + 'apikey': self._api_key, + } + params.update(extra) + try: + resp = http_requests.post(f"{self._url}/api", params=params, files=files, + timeout=self.DEFAULT_TIMEOUT) + if not resp.ok: + logger.warning("SABnzbd POST mode=%s returned HTTP %s", mode, resp.status_code) + return None + return resp.json() + except http_requests.exceptions.RequestException as e: + logger.error("SABnzbd POST mode=%s failed: %s", mode, e) + return None + except ValueError as e: + logger.error("SABnzbd POST mode=%s response was not JSON: %s", mode, e) + return None + + async def add_nzb( + self, + url_or_bytes: Union[str, bytes], + category: str = "soulsync", + save_path: Optional[str] = None, + ) -> Optional[str]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, self._add_nzb_sync, url_or_bytes, category, save_path + ) + + def _add_nzb_sync( + self, + url_or_bytes: Union[str, bytes], + category: str, + save_path: Optional[str], + ) -> Optional[str]: + cat = category or self._category + if isinstance(url_or_bytes, bytes): + files = {'name': ('soulsync.nzb', url_or_bytes, 'application/x-nzb')} + data = self._post_sync('addfile', files=files, cat=cat) + else: + data = self._call_sync('addurl', name=url_or_bytes, cat=cat) + if not data or not data.get('status'): + return None + ids = data.get('nzo_ids') or [] + return ids[0] if ids else None + + async def get_status(self, job_id: str) -> Optional[UsenetStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_status_sync, job_id) + + def _get_status_sync(self, job_id: str) -> Optional[UsenetStatus]: + # Check active queue first; if not found, fall back to history. + for status in self._get_all_sync(): + if status.id == job_id: + return status + return None + + async def get_all(self) -> List[UsenetStatus]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._get_all_sync) + + def _get_all_sync(self) -> List[UsenetStatus]: + out: List[UsenetStatus] = [] + # Active queue + queue = self._call_sync('queue') + if queue and isinstance(queue.get('queue'), dict): + for slot in queue['queue'].get('slots', []) or []: + out.append(self._parse_queue_slot(slot)) + # History — completed / failed jobs SAB still tracks + history = self._call_sync('history', limit=50) + if history and isinstance(history.get('history'), dict): + for slot in history['history'].get('slots', []) or []: + out.append(self._parse_history_slot(slot)) + return out + + def _parse_queue_slot(self, slot: dict) -> UsenetStatus: + try: + percentage = float(slot.get('percentage') or 0.0) + except (TypeError, ValueError): + percentage = 0.0 + progress = percentage / 100.0 + # mb / mbleft are strings of MB values in SAB's queue API. + size_mb = self._safe_float(slot.get('mb')) + left_mb = self._safe_float(slot.get('mbleft')) + size_bytes = int(size_mb * 1024 * 1024) if size_mb else 0 + downloaded_bytes = int((size_mb - left_mb) * 1024 * 1024) if size_mb and left_mb is not None else 0 + # ``timeleft`` is HH:MM:SS — convert to seconds. + eta = self._parse_timeleft(slot.get('timeleft')) + return UsenetStatus( + id=str(slot.get('nzo_id') or ''), + name=slot.get('filename') or slot.get('name') or '', + state=_map_state(slot.get('status') or ''), + progress=max(0.0, min(progress, 1.0)), + size=size_bytes, + downloaded=max(0, downloaded_bytes), + download_speed=0, # queue endpoint doesn't include per-slot speed + eta=eta, + category=slot.get('cat'), + ) + + def _parse_history_slot(self, slot: dict) -> UsenetStatus: + # History entries are post-download — progress is 1.0 unless failed. + sab_state = (slot.get('status') or '').lower() + is_failed = sab_state == 'failed' + return UsenetStatus( + id=str(slot.get('nzo_id') or ''), + name=slot.get('name') or '', + state='failed' if is_failed else 'completed', + progress=0.0 if is_failed else 1.0, + size=int(slot.get('bytes') or 0), + downloaded=int(slot.get('bytes') or 0) if not is_failed else 0, + download_speed=0, + save_path=slot.get('storage') or slot.get('path'), + category=slot.get('category'), + error=slot.get('fail_message') if is_failed else None, + ) + + @staticmethod + def _safe_float(value) -> Optional[float]: + if value is None or value == '': + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + @staticmethod + def _parse_timeleft(value) -> Optional[int]: + if not value or not isinstance(value, str): + return None + parts = value.split(':') + try: + if len(parts) == 3: + h, m, s = parts + return int(h) * 3600 + int(m) * 60 + int(s) + if len(parts) == 2: + m, s = parts + return int(m) * 60 + int(s) + except ValueError: + return None + return None + + async def remove(self, job_id: str, delete_files: bool = False) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._remove_sync, job_id, delete_files) + + def _remove_sync(self, job_id: str, delete_files: bool) -> bool: + # SAB deletes from queue or history depending on where the job is. + # We try queue first; if SAB reports no-op, fall through to history. + params = {'name': 'delete', 'value': job_id} + if delete_files: + params['del_files'] = 1 + data = self._call_sync('queue', **params) + if data and data.get('status'): + return True + # History delete + history_params = {'name': 'delete', 'value': job_id} + if delete_files: + history_params['del_files'] = 1 + data = self._call_sync('history', **history_params) + return bool(data and data.get('status')) + + async def pause(self, job_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._pause_sync, job_id) + + def _pause_sync(self, job_id: str) -> bool: + data = self._call_sync('queue', name='pause', value=job_id) + return bool(data and data.get('status')) + + async def resume(self, job_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._resume_sync, job_id) + + def _resume_sync(self, job_id: str) -> bool: + data = self._call_sync('queue', name='resume', value=job_id) + return bool(data and data.get('status')) diff --git a/core/watchlist/source_picker.py b/core/watchlist/source_picker.py index 666a5a91..0c40a21f 100644 --- a/core/watchlist/source_picker.py +++ b/core/watchlist/source_picker.py @@ -28,6 +28,7 @@ SOURCE_ID_COLUMNS = ( ('itunes', 'itunes_artist_id'), ('deezer', 'deezer_id'), ('discogs', 'discogs_id'), + ('musicbrainz', 'musicbrainz_id'), ) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index c90e95bd..9fdb967b 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -24,6 +24,19 @@ from core.wishlist_service import get_wishlist_service from core.matching_engine import MusicMatchingEngine from utils.logging_config import get_logger + +def _mark_personalized_kinds_stale(database, kinds, profile_id=1): + """Module-level helper so the inline call sites stay tiny. + + Constructs a PersonalizedPlaylistManager with the minimal deps + needed for `mark_kinds_stale` (database access only — no generator + dispatch required) and flips the is_stale flag for matching rows. + Best-effort: any exception is swallowed by the caller's try/except + since stale-flagging is non-critical for the scan itself.""" + from core.personalized.manager import PersonalizedPlaylistManager + mgr = PersonalizedPlaylistManager(database, deps=None) + return mgr.mark_kinds_stale(list(kinds), profile_id=profile_id) + logger = get_logger("watchlist_scanner") # Rate limiting constants for watchlist operations @@ -507,12 +520,8 @@ class WatchlistScanner: return list(get_source_priority(get_primary_source())) def _discovery_source_priority(self) -> List[str]: - """Return discovery sources in configured priority order. - - Discovery pool writes only support Spotify, iTunes, and Deezer IDs, so - we filter the broader metadata priority list down to those sources. - """ - return [source for source in self._watchlist_source_priority() if source in {'spotify', 'itunes', 'deezer'}] + """Return discovery sources in configured priority order.""" + return [source for source in self._watchlist_source_priority() if source in {'spotify', 'itunes', 'deezer', 'musicbrainz'}] @staticmethod def _artist_id_attribute_for_source(source: str) -> Optional[str]: @@ -522,6 +531,7 @@ class WatchlistScanner: 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id', + 'musicbrainz': 'musicbrainz_artist_id', }.get(source) @staticmethod @@ -531,6 +541,7 @@ class WatchlistScanner: 'spotify': 'similar_artist_spotify_id', 'itunes': 'similar_artist_itunes_id', 'deezer': 'similar_artist_deezer_id', + 'musicbrainz': 'similar_artist_musicbrainz_id', }.get(source) @staticmethod @@ -561,6 +572,9 @@ class WatchlistScanner: elif source == 'discogs': self.database.update_watchlist_discogs_id(watchlist_artist.id, source_id) watchlist_artist.discogs_artist_id = source_id + elif source == 'musicbrainz': + self.database.update_watchlist_musicbrainz_id(watchlist_artist.id, source_id) + watchlist_artist.musicbrainz_artist_id = source_id def _resolve_watchlist_artist_source_id(self, watchlist_artist: WatchlistArtist, source: str, client: Any) -> Optional[str]: """Resolve the artist ID for an exact source, searching by name if needed.""" @@ -891,7 +905,7 @@ class WatchlistScanner: cursor = conn.cursor() cursor.execute(""" SELECT id, artist_name, spotify_artist_id, itunes_artist_id, - deezer_artist_id, discogs_artist_id + deezer_artist_id, discogs_artist_id, musicbrainz_artist_id FROM watchlist_artists WHERE profile_id = ? AND (image_url IS NULL OR image_url = '' OR image_url = 'None' OR image_url NOT LIKE 'http%') @@ -946,7 +960,8 @@ class WatchlistScanner: if img: aid = (row['spotify_artist_id'] or row['itunes_artist_id'] - or row['deezer_artist_id'] or row['discogs_artist_id']) + or row['deezer_artist_id'] or row['discogs_artist_id'] + or row['musicbrainz_artist_id']) if aid: self.database.update_watchlist_artist_image(aid, img) else: @@ -979,7 +994,7 @@ class WatchlistScanner: """ # Per-artist metadata source override — if set, use that source first with fallback preferred = getattr(watchlist_artist, 'preferred_metadata_source', None) - if preferred and preferred in ('spotify', 'deezer', 'itunes', 'discogs'): + if preferred and preferred in ('spotify', 'deezer', 'itunes', 'discogs', 'musicbrainz'): source_priority = list(get_source_priority(preferred)) else: source_priority = self._watchlist_source_priority() @@ -1151,7 +1166,7 @@ class WatchlistScanner: # Keep this as a plain source list; resolve the client right before each use. providers_to_backfill = [ source for source in self._watchlist_source_priority() - if source in {'spotify', 'itunes', 'deezer', 'discogs'} + if source in {'spotify', 'itunes', 'deezer', 'discogs', 'musicbrainz'} ] for provider in providers_to_backfill: @@ -1208,6 +1223,7 @@ class WatchlistScanner: or artist.itunes_artist_id or artist.deezer_artist_id or artist.discogs_artist_id + or getattr(artist, 'musicbrainz_artist_id', None) or str(artist.id) ) @@ -1582,6 +1598,7 @@ class WatchlistScanner: 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id', + 'musicbrainz': 'musicbrainz_artist_id', }.get(provider) if not id_attr: @@ -1601,6 +1618,7 @@ class WatchlistScanner: 'itunes': self._match_to_itunes, 'deezer': self._match_to_deezer, 'discogs': self._match_to_discogs, + 'musicbrainz': self._match_to_musicbrainz, }.get(provider) update_fn = { @@ -1608,6 +1626,7 @@ class WatchlistScanner: 'itunes': self.database.update_watchlist_itunes_id, 'deezer': self.database.update_watchlist_deezer_id, 'discogs': self.database.update_watchlist_discogs_id, + 'musicbrainz': self.database.update_watchlist_musicbrainz_id, }.get(provider) if not match_fn or not update_fn: @@ -1767,6 +1786,17 @@ class WatchlistScanner: logger.warning(f"Could not match {artist_name} to Discogs: {e}") return None + def _match_to_musicbrainz(self, artist_name: str) -> Optional[str]: + """Match artist name to MusicBrainz ID using fuzzy name comparison.""" + try: + from core.metadata.registry import get_musicbrainz_client + client = get_musicbrainz_client() + results = client.search_artists(artist_name, limit=5) + return self._best_artist_match(results, artist_name) + except Exception as e: + logger.warning(f"Could not match {artist_name} to MusicBrainz: {e}") + return None + def _get_lookback_period_setting(self) -> str: """ Get the discovery lookback period setting from database. @@ -2203,6 +2233,8 @@ class WatchlistScanner: album_id = album.id album_release_date = album.release_date album_images = album.images if hasattr(album, 'images') else [] + if not album_images and hasattr(album, 'image_url') and album.image_url: + album_images = [{'url': album.image_url}] album_type = album.album_type if hasattr(album, 'album_type') else 'album' total_tracks = album.total_tracks if hasattr(album, 'total_tracks') else 0 album_artists = album.artists if hasattr(album, 'artists') else [] @@ -2359,6 +2391,7 @@ class WatchlistScanner: 'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', + 'musicbrainz': 'musicbrainz_id', } searched_source_ids = {} available_sources = [] @@ -2390,6 +2423,7 @@ class WatchlistScanner: 'spotify_id': None, 'itunes_id': None, 'deezer_id': None, + 'musicbrainz_id': None, 'image_url': None, 'genres': [], 'popularity': 0, @@ -2457,6 +2491,8 @@ class WatchlistScanner: return self.database.update_similar_artist_deezer_id(similar_artist_id, source_id) if source == 'itunes': return self.database.update_similar_artist_itunes_id(similar_artist_id, source_id) + if source == 'musicbrainz': + return self.database.update_similar_artist_musicbrainz_id(similar_artist_id, source_id) return False def _backfill_similar_artists_fallback_ids(self, source_artist_id: str, profile_id: int = 1) -> int: @@ -2467,7 +2503,7 @@ class WatchlistScanner: writable similar-artist ID columns. This keeps old cached rows usable when the active metadata provider changes. """ - backfill_sources = [source for source in self._discovery_source_priority() if source in {'itunes', 'deezer'}] + backfill_sources = [source for source in self._discovery_source_priority() if source in {'itunes', 'deezer', 'musicbrainz'}] if not backfill_sources: logger.debug("No fallback metadata providers available for similar-artist backfill") return 0 @@ -2569,14 +2605,18 @@ class WatchlistScanner: image_url=similar_artist.get('image_url'), genres=similar_artist.get('genres'), popularity=similar_artist.get('popularity', 0), - similar_artist_deezer_id=similar_artist.get('deezer_id') + similar_artist_deezer_id=similar_artist.get('deezer_id'), + similar_artist_musicbrainz_id=similar_artist.get('musicbrainz_id'), ) if success: stored_count += 1 - fallback_id = similar_artist.get('deezer_id') or similar_artist.get('itunes_id') - fallback_label = 'Deezer' if similar_artist.get('deezer_id') else 'iTunes' - logger.debug(f" #{rank}: {similar_artist['name']} (Spotify: {similar_artist.get('spotify_id')}, {fallback_label}: {fallback_id})") + ids = ', '.join( + f"{k}: {similar_artist.get(v)}" + for k, v in [('Spotify', 'spotify_id'), ('iTunes', 'itunes_id'), ('Deezer', 'deezer_id'), ('MB', 'musicbrainz_id')] + if similar_artist.get(v) + ) + logger.debug(f" #{rank}: {similar_artist['name']} ({ids})") except Exception as e: logger.warning(f"Error storing similar artist {similar_artist.get('name', 'Unknown')}: {e}") @@ -2672,6 +2712,8 @@ class WatchlistScanner: cache_callback = lambda found_id, artist_id=similar_artist.id: self.database.update_similar_artist_itunes_id(artist_id, found_id) elif source == 'deezer': cache_callback = lambda found_id, artist_id=similar_artist.id: self.database.update_similar_artist_deezer_id(artist_id, found_id) + elif source == 'musicbrainz': + cache_callback = lambda found_id, artist_id=similar_artist.id: self.database.update_similar_artist_musicbrainz_id(artist_id, found_id) artist_id = self._resolve_artist_id_for_source( source, @@ -2807,7 +2849,7 @@ class WatchlistScanner: track_data['deezer_track_id'] = track.get('id') track_data['deezer_album_id'] = album_data.get('id') track_data['deezer_artist_id'] = selected_artist_id - else: + elif selected_source == 'itunes': track_data['itunes_track_id'] = track.get('id') track_data['itunes_album_id'] = album_data.get('id') track_data['itunes_artist_id'] = selected_artist_id @@ -2941,7 +2983,7 @@ class WatchlistScanner: track_data['deezer_track_id'] = track.get('id') track_data['deezer_album_id'] = album_data.get('id') track_data['deezer_artist_id'] = artist_id_for_genres or '' - else: + elif db_source == 'itunes': track_data['itunes_track_id'] = track.get('id') track_data['itunes_album_id'] = album_data.get('id') track_data['itunes_artist_id'] = artist_id_for_genres or '' @@ -2980,6 +3022,22 @@ class WatchlistScanner: self.database.update_discovery_pool_timestamp(track_count=final_count, profile_id=profile_id) logger.info(f"Discovery pool now contains {final_count} total tracks (built over time)") + # Mark every personalized-playlist kind that draws from the + # discovery pool as stale so the playlist pipeline auto- + # regenerates snapshots on its next run. Without this the + # server playlists stay frozen even though the source pool + # just got fresh tracks. Best-effort — pool refresh succeeds + # even if the manager isn't wired (no personalized tables). + try: + _mark_personalized_kinds_stale( + self.database, + kinds=['hidden_gems', 'discovery_shuffle', 'popular_picks', + 'time_machine', 'genre_playlist', 'daily_mix'], + profile_id=profile_id, + ) + except Exception as e: # noqa: BLE001 — never abort scan for staleness flag + logger.debug("Failed to mark personalized kinds stale: %s", e) + # Cache recent albums for discovery page logger.info("Caching recent albums for discovery page...") if progress_callback: @@ -3147,7 +3205,7 @@ class WatchlistScanner: track_data['deezer_track_id'] = track['id'] track_data['deezer_album_id'] = album_data['id'] track_data['deezer_artist_id'] = selected_artist_id - else: + elif selected_source == 'itunes': track_data['itunes_track_id'] = track['id'] track_data['itunes_album_id'] = album_data['id'] track_data['itunes_artist_id'] = selected_artist_id @@ -3322,6 +3380,8 @@ class WatchlistScanner: selected_watchlist_id = artist.itunes_artist_id or artist_id elif source == 'deezer': selected_watchlist_id = getattr(artist, 'deezer_artist_id', None) or artist_id + elif source == 'musicbrainz': + selected_watchlist_id = artist_id break if not selected_source or not selected_artist_id or not selected_albums: @@ -3355,6 +3415,8 @@ class WatchlistScanner: cache_callback = lambda found_id, similar_id=artist.id: self.database.update_similar_artist_itunes_id(similar_id, found_id) elif source == 'deezer': cache_callback = lambda found_id, similar_id=artist.id: self.database.update_similar_artist_deezer_id(similar_id, found_id) + elif source == 'musicbrainz': + cache_callback = lambda found_id, similar_id=artist.id: self.database.update_similar_artist_musicbrainz_id(similar_id, found_id) artist_id = self._resolve_artist_id_for_source( source, @@ -3386,6 +3448,8 @@ class WatchlistScanner: selected_similar_id = artist.similar_artist_itunes_id or artist_id elif source == 'deezer': selected_similar_id = getattr(artist, 'similar_artist_deezer_id', None) or artist_id + elif source == 'musicbrainz': + selected_similar_id = getattr(artist, 'similar_artist_musicbrainz_id', None) or artist_id break if not selected_source or not selected_artist_id or not selected_albums: @@ -3660,6 +3724,14 @@ class WatchlistScanner: playlist_key = f'release_radar_{source}' self.database.save_curated_playlist(playlist_key, release_radar_tracks, profile_id=profile_id) logger.info(f"Release Radar ({source}) curated: {len(release_radar_tracks)} tracks") + # Flag personalized Fresh Tape snapshot as stale so the + # pipeline auto-regenerates it on the next run. + try: + _mark_personalized_kinds_stale( + self.database, kinds=['fresh_tape'], profile_id=profile_id, + ) + except Exception as e: # noqa: BLE001 + logger.debug("Fresh Tape stale-flag failed: %s", e) # 2. Curate Discovery Weekly - 50 tracks from discovery pool logger.info(f"Curating Discovery Weekly for {source}...") @@ -3735,6 +3807,12 @@ class WatchlistScanner: playlist_key = f'discovery_weekly_{source}' self.database.save_curated_playlist(playlist_key, discovery_weekly_tracks, profile_id=profile_id) logger.info(f"Discovery Weekly ({source}) curated: {len(discovery_weekly_tracks)} tracks") + try: + _mark_personalized_kinds_stale( + self.database, kinds=['archives'], profile_id=profile_id, + ) + except Exception as e: # noqa: BLE001 + logger.debug("Archives stale-flag failed: %s", e) # 3. "Because You Listen To" — personalized sections based on top played artists if profile['has_data']: diff --git a/core/wishlist/processing.py b/core/wishlist/processing.py index 542827b3..3a8cbf9d 100644 --- a/core/wishlist/processing.py +++ b/core/wishlist/processing.py @@ -232,13 +232,18 @@ def remove_tracks_already_in_library( log_prefix: str = "[Auto-Wishlist]", ) -> int: """Remove wishlist entries that are already present in the library.""" + from core.library import manual_library_match as _mlm + all_profiles = profiles_database.get_all_profiles() - cleanup_tracks = [] + # Carry (profile_id, track) so the match check can be profile-scoped. + cleanup_tracks: list[tuple[int, dict]] = [] for profile in all_profiles: - cleanup_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=profile["id"])) + pid = profile["id"] + for t in wishlist_service.get_wishlist_tracks_for_download(profile_id=pid): + cleanup_tracks.append((pid, t)) cleanup_removed = 0 - for track in cleanup_tracks: + for profile_id, track in cleanup_tracks: if skip_track_fn and skip_track_fn(track): continue @@ -250,6 +255,17 @@ def remove_tracks_already_in_library( if not track_name or not artists or not spotify_track_id: continue + # Manual match check — skip fuzzy search if user already linked this track. + if _mlm.get_match_for_track(music_database, profile_id, track, default_source='wishlist'): + try: + removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) + if removed: + cleanup_removed += 1 + logger.info(f"{log_prefix} [Manual Match] Skipped already-matched track: '{track_name}'") + except Exception as _mlm_err: + logger.error(f"{log_prefix} [Manual Match] Error removing track: {_mlm_err}") + continue + found_in_db = False matched_artist_name = '' for artist in artists: @@ -679,52 +695,14 @@ def automatic_wishlist_cleanup_after_db_update( logger.info(f"[Auto Cleanup] Found {len(wishlist_tracks)} tracks in wishlist") - removed_count = 0 - for track in wishlist_tracks: - track_name = track.get('name', '') - artists = track.get('artists', []) - spotify_track_id = track.get('spotify_track_id') or track.get('id') - track_album = track.get('album', {}).get('name') if isinstance(track.get('album'), dict) else track.get('album') - - if not track_name or not artists or not spotify_track_id: - continue - - found_in_db = False - for artist in artists: - if isinstance(artist, str): - artist_name = artist - elif isinstance(artist, dict) and 'name' in artist: - artist_name = artist['name'] - else: - artist_name = str(artist) - - try: - db_track, confidence = music_database.check_track_exists( - track_name, - artist_name, - confidence_threshold=0.7, - server_source=active_server, - album=track_album, - ) - - if db_track and confidence >= 0.7: - found_in_db = True - logger.info( - f"[Auto Cleanup] Track found in database: '{track_name}' by {artist_name} (confidence: {confidence:.2f})" - ) - break - except Exception as db_error: - logger.error(f"[Auto Cleanup] Error checking database for track '{track_name}': {db_error}") - continue - - if found_in_db: - try: - removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) - if removed: - removed_count += 1 - logger.info(f"[Auto Cleanup] Removed track from wishlist: '{track_name}' ({spotify_track_id})") - except Exception as remove_error: - logger.error(f"[Auto Cleanup] Error removing track from wishlist: {remove_error}") + removed_count = remove_tracks_already_in_library( + wishlist_service, + profiles_database, + music_database, + active_server, + logger=logger, + log_prefix="[Auto Cleanup]", + ) logger.info(f"[Auto Cleanup] Completed automatic cleanup: {removed_count} tracks removed from wishlist") return removed_count diff --git a/core/worker_utils.py b/core/worker_utils.py index d6f42523..e572c702 100644 --- a/core/worker_utils.py +++ b/core/worker_utils.py @@ -64,6 +64,17 @@ def set_album_api_track_count(cursor, album_id, count): (count, album_id), ) except Exception as e: + if "api_track_count" in str(e) and "no such column" in str(e).lower(): + try: + cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL") + cursor.execute( + "UPDATE albums SET api_track_count = ? WHERE id = ?", + (count, album_id), + ) + logger.info("Repaired missing api_track_count column while caching album track count") + return + except Exception as repair_error: + e = repair_error logger.warning( "Failed to cache api_track_count for album %s: %s", album_id, e ) diff --git a/database/music_database.py b/database/music_database.py index e6a1c35a..0171d88e 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -91,6 +91,7 @@ class WatchlistArtist: itunes_artist_id: Optional[str] = None # Cross-provider support deezer_artist_id: Optional[str] = None # Cross-provider support discogs_artist_id: Optional[str] = None # Cross-provider support + musicbrainz_artist_id: Optional[str] = None # Cross-provider support include_albums: bool = True include_eps: bool = True include_singles: bool = True @@ -118,6 +119,7 @@ class SimilarArtist: genres: Optional[List[str]] = None # Cached genres popularity: int = 0 # Cached popularity score similar_artist_deezer_id: Optional[str] = None # Deezer artist ID + similar_artist_musicbrainz_id: Optional[str] = None # MusicBrainz artist ID @dataclass class DiscoveryTrack: @@ -334,6 +336,7 @@ class MusicDatabase: itunes_artist_id TEXT, deezer_artist_id TEXT, discogs_artist_id TEXT, + musicbrainz_artist_id TEXT, artist_name TEXT NOT NULL, date_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP, last_scan_timestamp TIMESTAMP, @@ -407,6 +410,9 @@ class MusicDatabase: # Add Discogs enrichment columns (migration) self._add_discogs_columns(cursor) + # Add Amazon artist ID column (migration) + self._add_amazon_columns(cursor) + # Backfill match_status for rows that already have an external ID but # NULL status. Prevents enrichment workers from re-processing these # rows forever. Must run AFTER all *_match_status columns have been @@ -752,13 +758,48 @@ class MusicDatabase: ) """) + # Personalized-playlists subsystem schema (Group A + Group B + # unified storage). Idempotent — safe on every startup. + try: + from database.personalized_schema import ensure_personalized_schema + ensure_personalized_schema(conn) + except Exception as ps_err: + logger.error(f"Personalized-playlist schema init failed: {ps_err}") + + self._ensure_core_media_schema_columns(cursor) + conn.commit() logger.info("Database initialized successfully") except Exception as e: logger.error(f"Error initializing database: {e}") raise - + + self._init_manual_library_match_table() + + def _ensure_core_media_schema_columns(self, cursor): + """Repair required media-library columns that older migrations may miss. + + A few legacy migrations rebuild artists/albums/tracks in place. Newer + installs get these columns from CREATE TABLE, but upgraded databases can + occasionally miss one if a previous migration path failed or was marked + complete before the column existed. + """ + try: + cursor.execute("PRAGMA table_info(tracks)") + track_cols = {c[1] for c in cursor.fetchall()} + if track_cols and 'file_size' not in track_cols: + cursor.execute("ALTER TABLE tracks ADD COLUMN file_size INTEGER") + logger.info("Repaired missing file_size column on tracks table") + + cursor.execute("PRAGMA table_info(albums)") + album_cols = {c[1] for c in cursor.fetchall()} + if album_cols and 'api_track_count' not in album_cols: + cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL") + logger.info("Repaired missing api_track_count column on albums table") + except Exception as e: + logger.error("Error repairing core media schema columns: %s", e) + def _add_mirrored_playlist_explored_column(self, cursor): """Add explored_at column to mirrored_playlists to persist explore badge.""" try: @@ -851,6 +892,9 @@ class MusicDatabase: if 'server_source' not in tracks_columns: cursor.execute("ALTER TABLE tracks ADD COLUMN server_source TEXT DEFAULT 'plex'") logger.info("Added server_source column to tracks table") + if 'disc_number' not in tracks_columns: + cursor.execute("ALTER TABLE tracks ADD COLUMN disc_number INTEGER DEFAULT 1") + logger.info("Added disc_number column to tracks table") # Create indexes for server_source columns for performance cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_server_source ON artists (server_source)") @@ -1158,6 +1202,10 @@ class MusicDatabase: cursor.execute("ALTER TABLE similar_artists ADD COLUMN similar_artist_deezer_id TEXT") logger.info("Added similar_artist_deezer_id column to similar_artists table") + if 'similar_artist_musicbrainz_id' not in similar_artists_columns: + cursor.execute("ALTER TABLE similar_artists ADD COLUMN similar_artist_musicbrainz_id TEXT") + logger.info("Added similar_artist_musicbrainz_id column to similar_artists table") + # Migration: Add iTunes columns to recent_releases for dual-source discovery cursor.execute("PRAGMA table_info(recent_releases)") recent_releases_columns = [column[1] for column in cursor.fetchall()] @@ -1272,6 +1320,8 @@ class MusicDatabase: source_artist_id TEXT NOT NULL, similar_artist_spotify_id TEXT, similar_artist_itunes_id TEXT, + similar_artist_deezer_id TEXT, + similar_artist_musicbrainz_id TEXT, similar_artist_name TEXT NOT NULL, similarity_rank INTEGER DEFAULT 1, occurrence_count INTEGER DEFAULT 1, @@ -1282,8 +1332,10 @@ class MusicDatabase: migration_cursor.execute(""" INSERT OR IGNORE INTO similar_artists_new (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_deezer_id, similar_artist_musicbrainz_id, similar_artist_name, similarity_rank, occurrence_count, last_updated) SELECT source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_deezer_id, similar_artist_musicbrainz_id, similar_artist_name, similarity_rank, occurrence_count, last_updated FROM similar_artists """) @@ -1296,6 +1348,7 @@ class MusicDatabase: cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_itunes ON similar_artists (similar_artist_itunes_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_musicbrainz ON similar_artists (similar_artist_musicbrainz_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_occurrence ON similar_artists (occurrence_count)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_name ON similar_artists (similar_artist_name)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_spotify_track ON discovery_pool (spotify_track_id)") @@ -1476,6 +1529,7 @@ class MusicDatabase: itunes_artist_id TEXT, deezer_artist_id TEXT, discogs_artist_id TEXT, + musicbrainz_artist_id TEXT, image_url TEXT, genres TEXT, source_services TEXT DEFAULT '[]', @@ -1492,6 +1546,10 @@ class MusicDatabase: """) cursor.execute("CREATE INDEX IF NOT EXISTS idx_lap_profile ON liked_artists_pool (profile_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_lap_status ON liked_artists_pool (profile_id, match_status)") + cursor.execute("PRAGMA table_info(liked_artists_pool)") + liked_artist_columns = {column[1] for column in cursor.fetchall()} + if 'musicbrainz_artist_id' not in liked_artist_columns: + cursor.execute("ALTER TABLE liked_artists_pool ADD COLUMN musicbrainz_artist_id TEXT") # Liked albums pool — aggregated saved/liked albums from connected services cursor.execute(""" @@ -1623,6 +1681,14 @@ class MusicDatabase: cursor.execute("ALTER TABLE watchlist_artists ADD COLUMN discogs_artist_id TEXT") logger.info("Added discogs_artist_id column to watchlist_artists table for cross-provider support") + if 'amazon_artist_id' not in columns: + cursor.execute("ALTER TABLE watchlist_artists ADD COLUMN amazon_artist_id TEXT") + logger.info("Added amazon_artist_id column to watchlist_artists table for Amazon Music support") + + if 'musicbrainz_artist_id' not in columns: + cursor.execute("ALTER TABLE watchlist_artists ADD COLUMN musicbrainz_artist_id TEXT") + logger.info("Added musicbrainz_artist_id column to watchlist_artists table for MusicBrainz support") + except Exception as e: logger.error(f"Error adding itunes_artist_id column to watchlist_artists: {e}") # Don't raise - this is a migration, database can still function @@ -1712,6 +1778,7 @@ class MusicDatabase: itunes_artist_id TEXT, deezer_artist_id TEXT, discogs_artist_id TEXT, + musicbrainz_artist_id TEXT, profile_id INTEGER DEFAULT 1, UNIQUE(profile_id, spotify_artist_id), UNIQUE(profile_id, itunes_artist_id) @@ -1739,7 +1806,8 @@ class MusicDatabase: lookback_days INTEGER DEFAULT NULL, itunes_artist_id TEXT, deezer_artist_id TEXT, - discogs_artist_id TEXT + discogs_artist_id TEXT, + musicbrainz_artist_id TEXT ) """) @@ -1751,7 +1819,8 @@ class MusicDatabase: 'include_albums', 'include_eps', 'include_singles', 'include_live', 'include_remixes', 'include_acoustic', 'include_compilations', 'include_instrumentals', 'lookback_days', - 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', 'profile_id'] + 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', + 'musicbrainz_artist_id', 'profile_id'] shared_cols = [c for c in new_cols if c in old_cols] cols_str = ', '.join(shared_cols) cursor.execute(f"INSERT INTO watchlist_artists_new ({cols_str}) SELECT {cols_str} FROM watchlist_artists") @@ -2027,6 +2096,55 @@ class MusicDatabase: except Exception as e: logger.error(f"Error adding Discogs columns: {e}") + def _add_amazon_columns(self, cursor): + """Add Amazon enrichment tracking columns to artists, albums, and tracks.""" + try: + # --- Artists --- + cursor.execute("PRAGMA table_info(artists)") + artists_columns = [column[1] for column in cursor.fetchall()] + + if 'amazon_id' not in artists_columns: + cursor.execute("ALTER TABLE artists ADD COLUMN amazon_id TEXT") + if 'amazon_match_status' not in artists_columns: + cursor.execute("ALTER TABLE artists ADD COLUMN amazon_match_status TEXT") + if 'amazon_last_attempted' not in artists_columns: + cursor.execute("ALTER TABLE artists ADD COLUMN amazon_last_attempted TIMESTAMP") + + cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_amazon_id ON artists (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_amazon_status ON artists (amazon_match_status)") + + # --- Albums --- + cursor.execute("PRAGMA table_info(albums)") + albums_columns = [column[1] for column in cursor.fetchall()] + + if 'amazon_id' not in albums_columns: + cursor.execute("ALTER TABLE albums ADD COLUMN amazon_id TEXT") + if 'amazon_match_status' not in albums_columns: + cursor.execute("ALTER TABLE albums ADD COLUMN amazon_match_status TEXT") + if 'amazon_last_attempted' not in albums_columns: + cursor.execute("ALTER TABLE albums ADD COLUMN amazon_last_attempted TIMESTAMP") + + cursor.execute("CREATE INDEX IF NOT EXISTS idx_albums_amazon_id ON albums (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_albums_amazon_status ON albums (amazon_match_status)") + + # --- Tracks --- + cursor.execute("PRAGMA table_info(tracks)") + tracks_columns = [column[1] for column in cursor.fetchall()] + + if 'amazon_id' not in tracks_columns: + cursor.execute("ALTER TABLE tracks ADD COLUMN amazon_id TEXT") + if 'amazon_match_status' not in tracks_columns: + cursor.execute("ALTER TABLE tracks ADD COLUMN amazon_match_status TEXT") + if 'amazon_last_attempted' not in tracks_columns: + cursor.execute("ALTER TABLE tracks ADD COLUMN amazon_last_attempted TIMESTAMP") + + cursor.execute("CREATE INDEX IF NOT EXISTS idx_tracks_amazon_id ON tracks (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_tracks_amazon_status ON tracks (amazon_match_status)") + + logger.info("Amazon columns added/verified successfully") + except Exception as e: + logger.error(f"Error adding Amazon columns: {e}") + def _backfill_match_status_for_existing_ids(self, cursor): """Set `_match_status = 'matched'` for rows that already have a populated external ID but NULL match_status. @@ -2545,6 +2663,7 @@ class MusicDatabase: itunes_artist_id TEXT, deezer_artist_id TEXT, discogs_artist_id TEXT, + musicbrainz_artist_id TEXT, profile_id INTEGER DEFAULT 1, UNIQUE(profile_id, spotify_artist_id), UNIQUE(profile_id, itunes_artist_id) @@ -2557,7 +2676,8 @@ class MusicDatabase: 'include_albums', 'include_eps', 'include_singles', 'include_live', 'include_remixes', 'include_acoustic', 'include_compilations', 'include_instrumentals', 'lookback_days', - 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', 'profile_id'] + 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', + 'musicbrainz_artist_id', 'profile_id'] shared_cols = [c for c in new_cols if c in col_names] cols_str = ', '.join(shared_cols) @@ -2889,6 +3009,7 @@ class MusicDatabase: similar_artist_spotify_id TEXT, similar_artist_itunes_id TEXT, similar_artist_deezer_id TEXT, + similar_artist_musicbrainz_id TEXT, similar_artist_name TEXT NOT NULL, similarity_rank INTEGER DEFAULT 1, occurrence_count INTEGER DEFAULT 1, @@ -2905,7 +3026,8 @@ class MusicDatabase: new_cols = ['id', 'source_artist_id', 'similar_artist_spotify_id', 'similar_artist_itunes_id', 'similar_artist_deezer_id', - 'similar_artist_name', 'similarity_rank', 'occurrence_count', + 'similar_artist_musicbrainz_id', 'similar_artist_name', + 'similarity_rank', 'occurrence_count', 'last_updated', 'image_url', 'genres', 'popularity', 'metadata_updated_at', 'last_featured', 'profile_id'] shared_cols = [c for c in new_cols if c in old_cols] @@ -4070,6 +4192,172 @@ class MusicDatabase: except Exception as e: logger.error(f"Error creating repair worker v2 tables: {e}") + def _init_manual_library_match_table(self): + """Create manual_library_track_matches table and indexes.""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + CREATE TABLE IF NOT EXISTS manual_library_track_matches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + profile_id INTEGER DEFAULT 1, + source TEXT NOT NULL, + source_track_id TEXT NOT NULL, + source_title TEXT, + source_artist TEXT, + source_album TEXT, + source_context_json TEXT, + server_source TEXT DEFAULT '', + library_track_id INTEGER NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(profile_id, source, source_track_id, server_source) + ) + """) + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_mltm_lookup + ON manual_library_track_matches (profile_id, source, source_track_id, server_source) + """) + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_mltm_lib_track + ON manual_library_track_matches (library_track_id) + """) + except Exception as e: + logger.error(f"Error creating manual_library_track_matches table: {e}") + + def save_manual_library_match(self, profile_id: int, source: str, source_track_id: str, + library_track_id: int, **meta) -> bool: + """Insert or replace a manual match. meta keys: source_title, source_artist, + source_album, source_context_json, server_source.""" + try: + with self._get_connection() as conn: + conn.execute(""" + INSERT INTO manual_library_track_matches + (profile_id, source, source_track_id, library_track_id, + source_title, source_artist, source_album, + source_context_json, server_source, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(profile_id, source, source_track_id, server_source) + DO UPDATE SET + library_track_id = excluded.library_track_id, + source_title = excluded.source_title, + source_artist = excluded.source_artist, + source_album = excluded.source_album, + source_context_json = excluded.source_context_json, + updated_at = CURRENT_TIMESTAMP + """, ( + profile_id, source, source_track_id, library_track_id, + meta.get('source_title'), meta.get('source_artist'), + meta.get('source_album'), meta.get('source_context_json'), + meta.get('server_source', ''), + )) + return True + except Exception as e: + logger.error(f"save_manual_library_match error: {e}") + return False + + def get_manual_library_match(self, profile_id: int, source: str, + source_track_id: str, server_source: str = '') -> Optional[Dict[str, Any]]: + """Return match row dict or None.""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT * FROM manual_library_track_matches + WHERE profile_id = ? AND source = ? AND source_track_id = ? AND server_source = ? + """, (profile_id, source, source_track_id, server_source)) + row = cursor.fetchone() + return dict(row) if row else None + except Exception as e: + logger.error(f"get_manual_library_match error: {e}") + return None + + def find_manual_library_match_by_source_track_id(self, profile_id: int, + source_track_id: str, + server_source: str = '') -> Optional[Dict[str, Any]]: + """Return a manual match for this source track ID across source labels. + + The UI may save a match from sync history as ``mirrored`` while the + wishlist/download flow later sees the same track under ``wishlist`` or + the provider name. The source remains useful metadata, but the stored + track ID is the stable identity we need to honor. + """ + if not source_track_id: + return None + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT * FROM manual_library_track_matches + WHERE profile_id = ? + AND source_track_id = ? + AND (server_source = ? OR server_source = '') + ORDER BY + CASE WHEN server_source = ? THEN 0 ELSE 1 END, + updated_at DESC + LIMIT 1 + """, (profile_id, source_track_id, server_source or '', server_source or '')) + row = cursor.fetchone() + return dict(row) if row else None + except Exception as e: + logger.error(f"find_manual_library_match_by_source_track_id error: {e}") + return None + + def find_manual_library_match_by_metadata(self, profile_id: int, + source_title: str, + source_artist: str, + server_source: str = '') -> Optional[Dict[str, Any]]: + """Return a manual match by title/artist when provider IDs differ.""" + if not source_title or not source_artist: + return None + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT * FROM manual_library_track_matches + WHERE profile_id = ? + AND source_title = ? COLLATE NOCASE + AND source_artist = ? COLLATE NOCASE + AND (server_source = ? OR server_source = '') + ORDER BY + CASE WHEN server_source = ? THEN 0 ELSE 1 END, + updated_at DESC + LIMIT 1 + """, (profile_id, source_title, source_artist, server_source or '', server_source or '')) + row = cursor.fetchone() + return dict(row) if row else None + except Exception as e: + logger.error(f"find_manual_library_match_by_metadata error: {e}") + return None + + def delete_manual_library_match(self, match_id: int, profile_id: int) -> bool: + """Delete match by PK id, scoped to profile_id.""" + try: + with self._get_connection() as conn: + conn.execute(""" + DELETE FROM manual_library_track_matches WHERE id = ? AND profile_id = ? + """, (match_id, profile_id)) + return True + except Exception as e: + logger.error(f"delete_manual_library_match error: {e}") + return False + + def list_manual_library_matches(self, profile_id: int, limit: int = 100) -> List[Dict[str, Any]]: + """Return matches for profile ordered by updated_at DESC.""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT * FROM manual_library_track_matches + WHERE profile_id = ? + ORDER BY updated_at DESC + LIMIT ? + """, (profile_id, limit)) + return [dict(r) for r in cursor.fetchall()] + except Exception as e: + logger.error(f"list_manual_library_matches error: {e}") + return [] + # ── Profile CRUD ────────────────────────────────────────────────── def get_all_profiles(self) -> List[Dict[str, Any]]: @@ -4303,45 +4591,72 @@ class MusicDatabase: # VACUUM to actually shrink the database file and reclaim disk space logger.info("Vacuuming database to reclaim disk space...") - cursor.execute("VACUUM") + self._vacuum_best_effort(cursor) logger.info("All database data cleared and file compacted") except Exception as e: logger.error(f"Error clearing database: {e}") raise + + def _vacuum_best_effort(self, cursor): + """Run VACUUM without making the caller fail if compaction hiccups.""" + try: + cursor.execute("VACUUM") + except Exception as e: + logger.warning( + "Database VACUUM failed after data was already cleared; continuing without compaction: %s", + e, + ) + + @staticmethod + def _is_transient_sqlite_io_error(exc: Exception) -> bool: + return "disk i/o error" in str(exc).lower() def clear_server_data(self, server_source: str): """Clear data for specific server only (server-aware full refresh)""" - try: - with self._get_connection() as conn: - cursor = conn.cursor() - - # Delete only data from the specified server - # Order matters: tracks -> albums -> artists (foreign key constraints) - cursor.execute("DELETE FROM tracks WHERE server_source = ?", (server_source,)) - tracks_deleted = cursor.rowcount - - cursor.execute("DELETE FROM albums WHERE server_source = ?", (server_source,)) - albums_deleted = cursor.rowcount - - cursor.execute("DELETE FROM artists WHERE server_source = ?", (server_source,)) - artists_deleted = cursor.rowcount - - conn.commit() - - # Only VACUUM if we deleted a significant amount of data - if tracks_deleted > 1000 or albums_deleted > 100: - logger.info("Vacuuming database to reclaim disk space...") - cursor.execute("VACUUM") - - logger.info(f"Cleared {server_source} data: {artists_deleted} artists, {albums_deleted} albums, {tracks_deleted} tracks") - - # Note: Watchlist and wishlist are preserved as they are server-agnostic - - except Exception as e: - logger.error(f"Error clearing {server_source} database data: {e}") - raise + for attempt in range(2): + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + # Delete only data from the specified server + # Order matters: tracks -> albums -> artists (foreign key constraints) + cursor.execute("DELETE FROM tracks WHERE server_source = ?", (server_source,)) + tracks_deleted = cursor.rowcount + + cursor.execute("DELETE FROM albums WHERE server_source = ?", (server_source,)) + albums_deleted = cursor.rowcount + + cursor.execute("DELETE FROM artists WHERE server_source = ?", (server_source,)) + artists_deleted = cursor.rowcount + + conn.commit() + + # Only VACUUM if we deleted a significant amount of data + if tracks_deleted > 1000 or albums_deleted > 100: + logger.info("Vacuuming database to reclaim disk space...") + self._vacuum_best_effort(cursor) + + logger.info( + f"Cleared {server_source} data: {artists_deleted} artists, " + f"{albums_deleted} albums, {tracks_deleted} tracks" + ) + + # Note: Watchlist and wishlist are preserved as they are server-agnostic + return + + except Exception as e: + if self._is_transient_sqlite_io_error(e) and attempt == 0: + logger.warning( + "Transient disk I/O error clearing %s database data; retrying once: %s", + server_source, + e, + ) + time.sleep(0.25) + continue + logger.error(f"Error clearing {server_source} database data: {e}") + raise def cleanup_orphaned_records(self) -> Dict[str, int]: """Remove artists and albums that have no associated tracks""" @@ -5236,6 +5551,25 @@ class MusicDatabase: return True except Exception as e: + error_text = str(e).lower() + if ( + 'file_size' in error_text + and ('no such column' in error_text or 'no column named' in error_text) + and retry_count < max_retries - 1 + ): + try: + repair_conn = conn if 'conn' in locals() else self._get_connection() + repair_cursor = repair_conn.cursor() + self._ensure_core_media_schema_columns(repair_cursor) + repair_conn.commit() + if repair_conn is not conn: + repair_conn.close() + retry_count += 1 + logger.info("Repaired missing file_size column while importing media track; retrying") + continue + except Exception as schema_error: + logger.error("Failed to repair tracks.file_size during track import: %s", schema_error) + retry_count += 1 if "database is locked" in str(e).lower() and retry_count < max_retries: logger.warning(f"Database locked on track '{getattr(track_obj, 'title', 'Unknown')}', retrying {retry_count}/{max_retries}...") @@ -6080,7 +6414,7 @@ class MusicDatabase: logger.error(f"Error fetching candidate tracks for {len(album_ids)} album IDs: {e}") return [] - def check_album_exists_with_completeness(self, title: str, artist: str, expected_track_count: Optional[int] = None, confidence_threshold: float = 0.8, server_source: Optional[str] = None, candidate_albums: Optional[List[DatabaseAlbum]] = None) -> Tuple[Optional[DatabaseAlbum], float, int, int, bool, List[str]]: + def check_album_exists_with_completeness(self, title: str, artist: str, expected_track_count: Optional[int] = None, confidence_threshold: float = 0.8, server_source: Optional[str] = None, candidate_albums: Optional[List[DatabaseAlbum]] = None, strict_discography_match: bool = False) -> Tuple[Optional[DatabaseAlbum], float, int, int, bool, List[str]]: """ Check if an album exists in the database with completeness information. Enhanced to handle edition matching (standard <-> deluxe variants). @@ -6092,7 +6426,7 @@ class MusicDatabase: """ try: # Try enhanced edition-aware matching first with expected track count for Smart Edition Matching - album, confidence = self.check_album_exists_with_editions(title, artist, confidence_threshold, expected_track_count, server_source, candidate_albums=candidate_albums) + album, confidence = self.check_album_exists_with_editions(title, artist, confidence_threshold, expected_track_count, server_source, candidate_albums=candidate_albums, strict_discography_match=strict_discography_match) if not album: return None, 0.0, 0, 0, False, [] @@ -6106,7 +6440,7 @@ class MusicDatabase: logger.error(f"Error checking album existence with completeness for '{title}' by '{artist}': {e}") return None, 0.0, 0, 0, False, [] - def check_album_exists_with_editions(self, title: str, artist: str, confidence_threshold: float = 0.8, expected_track_count: Optional[int] = None, server_source: Optional[str] = None, candidate_albums: Optional[List[DatabaseAlbum]] = None) -> Tuple[Optional[DatabaseAlbum], float]: + def check_album_exists_with_editions(self, title: str, artist: str, confidence_threshold: float = 0.8, expected_track_count: Optional[int] = None, server_source: Optional[str] = None, candidate_albums: Optional[List[DatabaseAlbum]] = None, strict_discography_match: bool = False) -> Tuple[Optional[DatabaseAlbum], float]: """ Enhanced album existence check that handles edition variants. Matches standard albums with deluxe/platinum/special editions and vice versa. @@ -6129,7 +6463,7 @@ class MusicDatabase: # per-variation SQL widening that the legacy path does. logger.debug(f"Edition matching for '{title}' by '{artist}': batched against {len(candidate_albums)} candidates") for album in candidate_albums: - confidence = self._calculate_album_confidence(title, artist, album, expected_track_count) + confidence = self._calculate_album_confidence(title, artist, album, expected_track_count, strict_discography_match=strict_discography_match) if confidence > best_confidence: best_confidence = confidence best_match = album @@ -6162,7 +6496,7 @@ class MusicDatabase: # Score each potential match with Smart Edition Matching for album in albums: - confidence = self._calculate_album_confidence(title, artist, album, expected_track_count) + confidence = self._calculate_album_confidence(title, artist, album, expected_track_count, strict_discography_match=strict_discography_match) logger.debug(f" '{album.title}' confidence: {confidence:.3f}") if confidence > best_confidence: @@ -6198,7 +6532,7 @@ class MusicDatabase: logger.debug(f" Found {len(artist_albums)} total albums for artist fallback") for album in artist_albums: - confidence = self._calculate_album_confidence(title, artist, album, expected_track_count) + confidence = self._calculate_album_confidence(title, artist, album, expected_track_count, strict_discography_match=strict_discography_match) if confidence > best_confidence: best_confidence = confidence best_match = album @@ -6217,7 +6551,7 @@ class MusicDatabase: try: title_only_albums = self.search_albums(title=title, artist="", limit=20, server_source=server_source) for album in title_only_albums: - confidence = self._calculate_album_confidence(title, artist, album, expected_track_count) + confidence = self._calculate_album_confidence(title, artist, album, expected_track_count, strict_discography_match=strict_discography_match) # Slightly penalize cross-artist matches to prefer same-artist when possible if confidence > best_confidence: best_confidence = confidence @@ -6334,7 +6668,7 @@ class MusicDatabase: return unique_variations - def _calculate_album_confidence(self, search_title: str, search_artist: str, db_album: DatabaseAlbum, expected_track_count: Optional[int] = None) -> float: + def _calculate_album_confidence(self, search_title: str, search_artist: str, db_album: DatabaseAlbum, expected_track_count: Optional[int] = None, strict_discography_match: bool = False) -> float: """Calculate confidence score for album match with Smart Edition Matching""" try: # Simple confidence based on string similarity @@ -6354,6 +6688,18 @@ class MusicDatabase: # Use the best title similarity best_title_similarity = max(title_similarity, clean_title_similarity, normalized_title_similarity) + if strict_discography_match and not self._passes_strict_discography_album_match( + search_title, + db_album.title, + title_similarity, + clean_title_similarity, + normalized_title_similarity, + expected_track_count, + db_album.track_count, + ): + logger.debug(" Strict discography match rejected: '%s' -> '%s'", search_title, db_album.title) + return 0.0 + # Log when normalized matching helps (only if it's the best score and better than others) if normalized_title_similarity == best_title_similarity and normalized_title_similarity > max(title_similarity, clean_title_similarity): logger.debug(f" Diacritic normalization improved match: '{search_title}' -> '{db_album.title}' (normalized: {normalized_title_similarity:.3f} vs raw: {title_similarity:.3f})") @@ -6390,6 +6736,92 @@ class MusicDatabase: except Exception as e: logger.error(f"Error calculating album confidence: {e}") return 0.0 + + def _passes_strict_discography_album_match( + self, + search_title: str, + db_title: str, + title_similarity: float, + clean_title_similarity: float, + normalized_title_similarity: float, + expected_track_count: Optional[int], + db_track_count: Optional[int], + ) -> bool: + """Guard artist-page owned status against generic soundtrack false positives.""" + if not self._is_soundtrack_like_album_title(search_title) and not self._is_soundtrack_like_album_title(db_title): + return True + + normalized_search_title = self._normalize_for_comparison(search_title) + normalized_db_title = self._normalize_for_comparison(db_title) + if normalized_search_title == normalized_db_title: + return True + + clean_search_title = self._normalize_for_comparison(self._clean_album_title_for_comparison(search_title)) + clean_db_title = self._normalize_for_comparison(self._clean_album_title_for_comparison(db_title)) + if clean_search_title and clean_search_title == clean_db_title: + return True + + best_title_similarity = max(title_similarity, clean_title_similarity, normalized_title_similarity) + search_tokens = self._distinctive_soundtrack_title_tokens(search_title) + db_tokens = self._distinctive_soundtrack_title_tokens(db_title) + if not search_tokens or not db_tokens: + return False + + shared_tokens = search_tokens & db_tokens + smaller_overlap = len(shared_tokens) / min(len(search_tokens), len(db_tokens)) + jaccard_overlap = len(shared_tokens) / len(search_tokens | db_tokens) + if smaller_overlap < 0.75 or jaccard_overlap < 0.55: + return False + + if expected_track_count and db_track_count and best_title_similarity < 0.9: + track_ratio = min(expected_track_count, db_track_count) / max(expected_track_count, db_track_count) + if track_ratio < 0.5: + return False + + return True + + def _is_soundtrack_like_album_title(self, title: str) -> bool: + title = (title or "").lower() + patterns = [ + r"\bsoundtrack\b", + r"\bscore\b", + r"\bost\b", + r"original\s+motion\s+picture", + r"music\s+from\s+(?:the\s+)?(?:motion\s+picture|film|movie|series|anime|tv|television)", + r"complete\s+recordings?", + ] + return any(re.search(pattern, title) for pattern in patterns) + + def _distinctive_soundtrack_title_tokens(self, title: str) -> set[str]: + normalized = self._normalize_for_comparison(title) + tokens = set(re.findall(r"[a-z0-9]+", normalized)) + noise = { + "album", + "anime", + "complete", + "deluxe", + "edition", + "film", + "from", + "motion", + "movie", + "music", + "official", + "original", + "ost", + "picture", + "recording", + "recordings", + "score", + "series", + "soundtrack", + "special", + "television", + "the", + "tv", + "version", + } + return {token for token in tokens if token not in noise and len(token) > 1} def _generate_track_title_variations(self, title: str) -> List[str]: """Generate variations of track title for better matching""" @@ -7040,6 +7472,16 @@ class MusicDatabase: logger.error("Cannot add track to wishlist: missing track ID") return False + from core.library import manual_library_match as _mlm + if _mlm.get_match_for_track(self, profile_id, spotify_track_data): + logger.info( + "Skipping wishlist add for manually matched track: '%s' (%s:%s)", + spotify_track_data.get('name', 'Unknown Track'), + spotify_track_data.get('provider') or spotify_track_data.get('source') or 'unknown', + track_id, + ) + return True + track_name = spotify_track_data.get('name', 'Unknown Track') artists = spotify_track_data.get('artists', []) if artists: @@ -7392,7 +7834,8 @@ class MusicDatabase: # Check if artist already exists by name (case-insensitive) for this profile cursor.execute(""" - SELECT id, spotify_artist_id, itunes_artist_id, deezer_artist_id, discogs_artist_id + SELECT id, spotify_artist_id, itunes_artist_id, deezer_artist_id, + discogs_artist_id, musicbrainz_artist_id FROM watchlist_artists WHERE LOWER(artist_name) = LOWER(?) AND profile_id = ? LIMIT 1 @@ -7405,7 +7848,13 @@ class MusicDatabase: if existing: # Artist already on watchlist — update with new source ID if missing - col_map = {'spotify': 'spotify_artist_id', 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id'} + col_map = { + 'spotify': 'spotify_artist_id', + 'itunes': 'itunes_artist_id', + 'deezer': 'deezer_artist_id', + 'discogs': 'discogs_artist_id', + 'musicbrainz': 'musicbrainz_artist_id', + } col = col_map.get(source) if col and not existing[col]: cursor.execute(f""" @@ -7441,6 +7890,13 @@ class MusicDatabase: VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, ?) """, (artist_id, artist_name, profile_id)) logger.info(f"Added artist '{artist_name}' to watchlist (Discogs ID: {artist_id}, profile: {profile_id})") + elif source == 'musicbrainz': + cursor.execute(""" + INSERT INTO watchlist_artists + (musicbrainz_artist_id, artist_name, date_added, updated_at, profile_id) + VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, ?) + """, (artist_id, artist_name, profile_id)) + logger.info(f"Added artist '{artist_name}' to watchlist (MusicBrainz ID: {artist_id}, profile: {profile_id})") else: cursor.execute(""" INSERT INTO watchlist_artists @@ -7457,7 +7913,7 @@ class MusicDatabase: return False def remove_artist_from_watchlist(self, artist_id: str, profile_id: int = 1) -> bool: - """Remove an artist from the watchlist (checks Spotify, iTunes, Deezer, and Discogs IDs)""" + """Remove an artist from the watchlist (checks cross-provider artist IDs)""" try: with self._get_connection() as conn: cursor = conn.cursor() @@ -7465,15 +7921,17 @@ class MusicDatabase: # Get artist name for logging (check all ID columns) cursor.execute(""" SELECT artist_name FROM watchlist_artists - WHERE (spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ?) AND profile_id = ? - """, (artist_id, artist_id, artist_id, artist_id, profile_id)) + WHERE (spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR musicbrainz_artist_id = ?) AND profile_id = ? + """, (artist_id, artist_id, artist_id, artist_id, artist_id, profile_id)) result = cursor.fetchone() artist_name = result['artist_name'] if result else "Unknown" cursor.execute(""" DELETE FROM watchlist_artists - WHERE (spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ?) AND profile_id = ? - """, (artist_id, artist_id, artist_id, artist_id, profile_id)) + WHERE (spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR musicbrainz_artist_id = ?) AND profile_id = ? + """, (artist_id, artist_id, artist_id, artist_id, artist_id, profile_id)) if cursor.rowcount > 0: conn.commit() @@ -7488,7 +7946,7 @@ class MusicDatabase: return False def is_artist_in_watchlist(self, artist_id: str, profile_id: int = 1, artist_name: str = None) -> bool: - """Check if an artist is currently in the watchlist (checks Spotify, iTunes, Deezer, Discogs IDs and name)""" + """Check if an artist is currently in the watchlist (checks cross-provider IDs and name)""" try: with self._get_connection() as conn: cursor = conn.cursor() @@ -7497,15 +7955,18 @@ class MusicDatabase: if artist_name: cursor.execute(""" SELECT 1 FROM watchlist_artists - WHERE (spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? OR LOWER(artist_name) = LOWER(?)) AND profile_id = ? + WHERE (spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR musicbrainz_artist_id = ? + OR LOWER(artist_name) = LOWER(?)) AND profile_id = ? LIMIT 1 - """, (artist_id, artist_id, artist_id, artist_id, artist_name, profile_id)) + """, (artist_id, artist_id, artist_id, artist_id, artist_id, artist_name, profile_id)) else: cursor.execute(""" SELECT 1 FROM watchlist_artists - WHERE (spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ?) AND profile_id = ? + WHERE (spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR musicbrainz_artist_id = ?) AND profile_id = ? LIMIT 1 - """, (artist_id, artist_id, artist_id, artist_id, profile_id)) + """, (artist_id, artist_id, artist_id, artist_id, artist_id, profile_id)) result = cursor.fetchone() return result is not None @@ -7527,7 +7988,7 @@ class MusicDatabase: # Build SELECT query based on existing columns base_columns = ['id', 'spotify_artist_id', 'artist_name', 'date_added', 'last_scan_timestamp', 'created_at', 'updated_at'] - optional_columns = ['image_url', 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', 'include_albums', 'include_eps', 'include_singles', + optional_columns = ['image_url', 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', 'musicbrainz_artist_id', 'include_albums', 'include_eps', 'include_singles', 'include_live', 'include_remixes', 'include_acoustic', 'include_compilations', 'include_instrumentals', 'lookback_days', 'preferred_metadata_source'] @@ -7556,6 +8017,7 @@ class MusicDatabase: itunes_artist_id = row['itunes_artist_id'] if 'itunes_artist_id' in existing_columns else None deezer_artist_id = row['deezer_artist_id'] if 'deezer_artist_id' in existing_columns else None discogs_artist_id = row['discogs_artist_id'] if 'discogs_artist_id' in existing_columns else None + musicbrainz_artist_id = row['musicbrainz_artist_id'] if 'musicbrainz_artist_id' in existing_columns else None include_albums = bool(row['include_albums']) if 'include_albums' in existing_columns else True include_eps = bool(row['include_eps']) if 'include_eps' in existing_columns else True include_singles = bool(row['include_singles']) if 'include_singles' in existing_columns else True @@ -7579,6 +8041,7 @@ class MusicDatabase: itunes_artist_id=itunes_artist_id, deezer_artist_id=deezer_artist_id, discogs_artist_id=discogs_artist_id, + musicbrainz_artist_id=musicbrainz_artist_id, include_albums=include_albums, include_eps=include_eps, include_singles=include_singles, @@ -7778,8 +8241,9 @@ class MusicDatabase: cursor.execute(""" UPDATE watchlist_artists SET image_url = ?, updated_at = CURRENT_TIMESTAMP - WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? - """, (image_url, artist_id, artist_id, artist_id, artist_id)) + WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR musicbrainz_artist_id = ? + """, (image_url, artist_id, artist_id, artist_id, artist_id, artist_id)) conn.commit() return cursor.rowcount > 0 @@ -7865,6 +8329,107 @@ class MusicDatabase: logger.error(f"Error updating watchlist Discogs ID: {e}") return False + def update_watchlist_musicbrainz_id(self, watchlist_id: int, musicbrainz_id: str) -> bool: + """Update the MusicBrainz artist ID for a watchlist artist (cross-provider support)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + UPDATE watchlist_artists + SET musicbrainz_artist_id = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (musicbrainz_id, watchlist_id)) + conn.commit() + logger.info(f"Updated MusicBrainz ID for watchlist artist {watchlist_id}: {musicbrainz_id}") + return cursor.rowcount > 0 + except Exception as e: + logger.error(f"Error updating watchlist MusicBrainz ID: {e}") + return False + + def backfill_watchlist_musicbrainz_ids_from_library(self, profile_id: int = 1) -> int: + """Copy existing library MusicBrainz artist IDs onto matching watchlist rows. + + The MusicBrainz enrichment worker writes IDs to ``artists.musicbrainz_id``. + Watchlist UI reads ``watchlist_artists.musicbrainz_artist_id``, so this + bridge lets existing enriched library matches show up as watchlist + MusicBrainz matches without waiting for a separate watchlist scan. + """ + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + UPDATE watchlist_artists + SET musicbrainz_artist_id = ( + SELECT a.musicbrainz_id + FROM artists a + WHERE a.musicbrainz_id IS NOT NULL + AND a.musicbrainz_id != '' + AND ( + LOWER(a.name) = LOWER(watchlist_artists.artist_name) + OR ( + watchlist_artists.spotify_artist_id IS NOT NULL + AND watchlist_artists.spotify_artist_id != '' + AND a.spotify_artist_id = watchlist_artists.spotify_artist_id + ) + OR ( + watchlist_artists.itunes_artist_id IS NOT NULL + AND watchlist_artists.itunes_artist_id != '' + AND a.itunes_artist_id = watchlist_artists.itunes_artist_id + ) + OR ( + watchlist_artists.deezer_artist_id IS NOT NULL + AND watchlist_artists.deezer_artist_id != '' + AND a.deezer_id = watchlist_artists.deezer_artist_id + ) + OR ( + watchlist_artists.discogs_artist_id IS NOT NULL + AND watchlist_artists.discogs_artist_id != '' + AND a.discogs_id = watchlist_artists.discogs_artist_id + ) + ) + LIMIT 1 + ), + updated_at = CURRENT_TIMESTAMP + WHERE profile_id = ? + AND (musicbrainz_artist_id IS NULL OR musicbrainz_artist_id = '') + AND EXISTS ( + SELECT 1 + FROM artists a + WHERE a.musicbrainz_id IS NOT NULL + AND a.musicbrainz_id != '' + AND ( + LOWER(a.name) = LOWER(watchlist_artists.artist_name) + OR ( + watchlist_artists.spotify_artist_id IS NOT NULL + AND watchlist_artists.spotify_artist_id != '' + AND a.spotify_artist_id = watchlist_artists.spotify_artist_id + ) + OR ( + watchlist_artists.itunes_artist_id IS NOT NULL + AND watchlist_artists.itunes_artist_id != '' + AND a.itunes_artist_id = watchlist_artists.itunes_artist_id + ) + OR ( + watchlist_artists.deezer_artist_id IS NOT NULL + AND watchlist_artists.deezer_artist_id != '' + AND a.deezer_id = watchlist_artists.deezer_artist_id + ) + OR ( + watchlist_artists.discogs_artist_id IS NOT NULL + AND watchlist_artists.discogs_artist_id != '' + AND a.discogs_id = watchlist_artists.discogs_artist_id + ) + ) + ) + """, (profile_id,)) + conn.commit() + if cursor.rowcount: + logger.info("Backfilled %s watchlist MusicBrainz artist IDs from library", cursor.rowcount) + return cursor.rowcount + except Exception as e: + logger.error(f"Error backfilling watchlist MusicBrainz IDs from library: {e}") + return 0 + def update_watchlist_artist_itunes_id(self, spotify_artist_id: str, itunes_id: str) -> bool: """Update the iTunes artist ID for a watchlist artist by Spotify ID (for cross-provider caching)""" try: @@ -7917,25 +8482,27 @@ class MusicDatabase: image_url: Optional[str] = None, genres: Optional[list] = None, popularity: int = 0, - similar_artist_deezer_id: Optional[str] = None) -> bool: - """Add or update a similar artist recommendation (supports Spotify, iTunes, and Deezer IDs)""" + similar_artist_deezer_id: Optional[str] = None, + similar_artist_musicbrainz_id: Optional[str] = None) -> bool: + """Add or update a similar artist recommendation.""" try: with self._get_connection() as conn: cursor = conn.cursor() genres_json = json.dumps(genres) if genres else None - # Use artist name as the unique key (allows storing both IDs for same artist) cursor.execute(""" INSERT INTO similar_artists - (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_deezer_id, similar_artist_name, + (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_deezer_id, similar_artist_musicbrainz_id, similar_artist_name, similarity_rank, occurrence_count, last_updated, profile_id, image_url, genres, popularity, metadata_updated_at) - VALUES (?, ?, ?, ?, ?, ?, 1, CURRENT_TIMESTAMP, ?, ?, ?, ?, CURRENT_TIMESTAMP) + VALUES (?, ?, ?, ?, ?, ?, ?, 1, CURRENT_TIMESTAMP, ?, ?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT(profile_id, source_artist_id, similar_artist_name) DO UPDATE SET similar_artist_spotify_id = COALESCE(excluded.similar_artist_spotify_id, similar_artist_spotify_id), similar_artist_itunes_id = COALESCE(excluded.similar_artist_itunes_id, similar_artist_itunes_id), similar_artist_deezer_id = COALESCE(excluded.similar_artist_deezer_id, similar_artist_deezer_id), + similar_artist_musicbrainz_id = COALESCE(excluded.similar_artist_musicbrainz_id, similar_artist_musicbrainz_id), similarity_rank = excluded.similarity_rank, occurrence_count = occurrence_count + 1, last_updated = CURRENT_TIMESTAMP, @@ -7943,7 +8510,8 @@ class MusicDatabase: genres = COALESCE(excluded.genres, genres), popularity = CASE WHEN excluded.popularity > 0 THEN excluded.popularity ELSE popularity END, metadata_updated_at = CASE WHEN excluded.image_url IS NOT NULL THEN CURRENT_TIMESTAMP ELSE metadata_updated_at END - """, (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_deezer_id, similar_artist_name, + """, (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_deezer_id, similar_artist_musicbrainz_id, similar_artist_name, similarity_rank, profile_id, image_url, genres_json, popularity)) conn.commit() @@ -7976,6 +8544,7 @@ class MusicDatabase: occurrence_count=row['occurrence_count'], last_updated=datetime.fromisoformat(row['last_updated']), similar_artist_deezer_id=row['similar_artist_deezer_id'] if 'similar_artist_deezer_id' in row.keys() else None, + similar_artist_musicbrainz_id=row['similar_artist_musicbrainz_id'] if 'similar_artist_musicbrainz_id' in row.keys() else None, ) for row in rows] except Exception as e: @@ -7985,11 +8554,14 @@ class MusicDatabase: def get_similar_artists_missing_fallback_ids(self, source_artist_id: str, fallback_source: str = 'itunes', profile_id: int = 1) -> List[SimilarArtist]: """Get similar artists missing fallback-provider IDs for backfill.""" try: - if fallback_source not in {'itunes', 'deezer'}: + if fallback_source not in {'itunes', 'deezer', 'musicbrainz'}: logger.error("Unsupported similar-artist fallback source: %s", fallback_source) return [] - col = 'similar_artist_deezer_id' if fallback_source == 'deezer' else 'similar_artist_itunes_id' + col = { + 'deezer': 'similar_artist_deezer_id', + 'musicbrainz': 'similar_artist_musicbrainz_id', + }.get(fallback_source, 'similar_artist_itunes_id') with self._get_connection() as conn: cursor = conn.cursor() @@ -8012,6 +8584,7 @@ class MusicDatabase: occurrence_count=row['occurrence_count'], last_updated=datetime.fromisoformat(row['last_updated']), similar_artist_deezer_id=row['similar_artist_deezer_id'] if 'similar_artist_deezer_id' in row.keys() else None, + similar_artist_musicbrainz_id=row['similar_artist_musicbrainz_id'] if 'similar_artist_musicbrainz_id' in row.keys() else None, ) for row in rows] except Exception as e: @@ -8056,6 +8629,25 @@ class MusicDatabase: logger.error(f"Error updating similar artist Deezer ID: {e}") return False + def update_similar_artist_musicbrainz_id(self, similar_artist_id: int, musicbrainz_id: str) -> bool: + """Update a similar artist's MusicBrainz ID (for backfill)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute(""" + UPDATE similar_artists + SET similar_artist_musicbrainz_id = ? + WHERE id = ? + """, (musicbrainz_id, similar_artist_id)) + + conn.commit() + return cursor.rowcount > 0 + + except Exception as e: + logger.error(f"Error updating similar artist MusicBrainz ID: {e}") + return False + def update_similar_artist_metadata(self, similar_artist_id: int, image_url: str = None, genres: list = None, popularity: int = None) -> bool: """Cache artist metadata (image, genres, popularity) to avoid repeated API calls""" @@ -8077,7 +8669,7 @@ class MusicDatabase: def update_similar_artist_metadata_by_external_id(self, external_id: str, source: str = 'spotify', image_url: str = None, genres: list = None, popularity: int = None) -> bool: - """Cache artist metadata by Spotify or iTunes ID (updates all rows for that artist)""" + """Cache artist metadata by external source ID (updates all rows for that artist).""" try: with self._get_connection() as conn: cursor = conn.cursor() @@ -8086,6 +8678,8 @@ class MusicDatabase: where_clause = "similar_artist_spotify_id = ?" elif source == 'deezer': where_clause = "similar_artist_deezer_id = ?" + elif source == 'musicbrainz': + where_clause = "similar_artist_musicbrainz_id = ?" else: where_clause = "similar_artist_itunes_id = ?" cursor.execute(f""" @@ -8139,9 +8733,16 @@ class MusicDatabase: logger.error(f"Error checking similar artists freshness: {e}") return False # Default to re-fetching on error - def get_top_similar_artists(self, limit: int = 50, profile_id: int = 1, require_source: str = None) -> List[SimilarArtist]: + def get_top_similar_artists( + self, + limit: int = 50, + profile_id: int = 1, + require_source: str = None, + exclude_library_server: str = None, + ) -> List[SimilarArtist]: """Get top similar artists excluding watchlist artists, with cycling support. - require_source: if set ('spotify','itunes','deezer'), only returns artists with that source ID.""" + require_source: if set, only returns artists with that source ID. + exclude_library_server: if set, also excludes artists already present in that media server.""" try: with self._get_connection() as conn: cursor = conn.cursor() @@ -8154,6 +8755,30 @@ class MusicDatabase: source_filter = "AND sa.similar_artist_itunes_id IS NOT NULL AND sa.similar_artist_itunes_id != ''" elif require_source == 'deezer': source_filter = "AND sa.similar_artist_deezer_id IS NOT NULL AND sa.similar_artist_deezer_id != ''" + elif require_source == 'musicbrainz': + source_filter = "AND sa.similar_artist_musicbrainz_id IS NOT NULL AND sa.similar_artist_musicbrainz_id != ''" + + library_artist_keys = None + sql_limit = limit + if exclude_library_server: + cursor.execute(""" + SELECT name, spotify_artist_id, itunes_artist_id, deezer_id, musicbrainz_id + FROM artists + WHERE server_source = ? + """, (exclude_library_server,)) + library_rows = cursor.fetchall() + library_artist_keys = { + 'spotify': {r['spotify_artist_id'] for r in library_rows if r['spotify_artist_id']}, + 'itunes': {r['itunes_artist_id'] for r in library_rows if r['itunes_artist_id']}, + 'deezer': {r['deezer_id'] for r in library_rows if r['deezer_id']}, + 'musicbrainz': {r['musicbrainz_id'] for r in library_rows if r['musicbrainz_id']}, + 'names': { + self._normalize_for_comparison(r['name']) + for r in library_rows + if r['name'] + }, + } + sql_limit = max(limit * 5, limit + 100) cursor.execute(f""" SELECT @@ -8162,6 +8787,7 @@ class MusicDatabase: MAX(sa.similar_artist_spotify_id) as similar_artist_spotify_id, MAX(sa.similar_artist_itunes_id) as similar_artist_itunes_id, MAX(sa.similar_artist_deezer_id) as similar_artist_deezer_id, + MAX(sa.similar_artist_musicbrainz_id) as similar_artist_musicbrainz_id, sa.similar_artist_name, AVG(sa.similarity_rank) as similarity_rank, SUM(sa.occurrence_count) as occurrence_count, @@ -8184,11 +8810,26 @@ class MusicDatabase: occurrence_count DESC, similarity_rank ASC LIMIT ? - """, (profile_id, profile_id, limit)) + """, (profile_id, profile_id, sql_limit)) rows = cursor.fetchall() results = [] for row in rows: + if library_artist_keys: + spotify_id = row['similar_artist_spotify_id'] + itunes_id = row['similar_artist_itunes_id'] if 'similar_artist_itunes_id' in row.keys() else None + deezer_id = row['similar_artist_deezer_id'] if 'similar_artist_deezer_id' in row.keys() else None + musicbrainz_id = row['similar_artist_musicbrainz_id'] if 'similar_artist_musicbrainz_id' in row.keys() else None + normalized_name = self._normalize_for_comparison(row['similar_artist_name']) + if ( + (spotify_id and spotify_id in library_artist_keys['spotify']) + or (itunes_id and itunes_id in library_artist_keys['itunes']) + or (deezer_id and deezer_id in library_artist_keys['deezer']) + or (musicbrainz_id and musicbrainz_id in library_artist_keys['musicbrainz']) + or (normalized_name and normalized_name in library_artist_keys['names']) + ): + continue + genres_raw = row['genres'] if 'genres' in row.keys() else None try: genres_list = json.loads(genres_raw) if genres_raw else None @@ -8200,6 +8841,7 @@ class MusicDatabase: similar_artist_spotify_id=row['similar_artist_spotify_id'], similar_artist_itunes_id=row['similar_artist_itunes_id'] if 'similar_artist_itunes_id' in row.keys() else None, similar_artist_deezer_id=row['similar_artist_deezer_id'] if 'similar_artist_deezer_id' in row.keys() else None, + similar_artist_musicbrainz_id=row['similar_artist_musicbrainz_id'] if 'similar_artist_musicbrainz_id' in row.keys() else None, similar_artist_name=row['similar_artist_name'], similarity_rank=int(row['similarity_rank']), occurrence_count=row['occurrence_count'], @@ -8208,6 +8850,8 @@ class MusicDatabase: genres=genres_list, popularity=row['popularity'] if 'popularity' in row.keys() else 0, )) + if len(results) >= limit: + break return results except Exception as e: @@ -8895,6 +9539,7 @@ class MusicDatabase: a.tidal_id, a.qobuz_id, a.soul_id, + a.amazon_id, a.server_source FROM artists a WHERE {where_clause} @@ -8994,6 +9639,7 @@ class MusicDatabase: 'tidal_id': row['tidal_id'], 'qobuz_id': row['qobuz_id'], 'soul_id': row['soul_id'], + 'amazon_id': row['amazon_id'], 'album_count': counts_map.get(row['id'], (0, 0))[0], 'track_count': counts_map.get(row['id'], (0, 0))[1], 'is_watched': bool(is_watched) @@ -9052,7 +9698,7 @@ class MusicDatabase: id, name, thumb_url, genres, server_source, musicbrainz_id, deezer_id, audiodb_id, discogs_id, spotify_artist_id, itunes_artist_id, lastfm_url, genius_url, - tidal_id, qobuz_id, soul_id, + tidal_id, qobuz_id, soul_id, amazon_id, lastfm_listeners, lastfm_playcount, lastfm_tags, lastfm_bio FROM artists WHERE id = ? @@ -9210,6 +9856,7 @@ class MusicDatabase: 'tidal_id': artist_row['tidal_id'], 'qobuz_id': artist_row['qobuz_id'], 'soul_id': artist_row['soul_id'], + 'amazon_id': artist_row['amazon_id'], 'lastfm_listeners': artist_row['lastfm_listeners'], 'lastfm_playcount': artist_row['lastfm_playcount'], 'lastfm_tags': artist_row['lastfm_tags'], @@ -9864,7 +10511,7 @@ class MusicDatabase: # Store all discovered source IDs (COALESCE preserves existing values) if all_ids: - for col in ('spotify_artist_id', 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id'): + for col in ('spotify_artist_id', 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', 'musicbrainz_artist_id'): val = all_ids.get(col) if val: set_parts.append(f"{col} = COALESCE({col}, ?)") diff --git a/database/personalized_schema.py b/database/personalized_schema.py new file mode 100644 index 00000000..27df8477 --- /dev/null +++ b/database/personalized_schema.py @@ -0,0 +1,159 @@ +"""Schema + migration helpers for the personalized-playlists subsystem. + +Pre-existing state (this PR replaces over time): +- Group A (Fresh Tape / The Archives / Seasonal Mix) lives in + `discovery_curated_playlists` (track_ids only) and + `curated_seasonal_playlists` (track_ids + seasonal_tracks join). + Read paths exist; refresh paths are tied to specific workers. +- Group B (Hidden Gems / Discovery Shuffle / Time Machine / Popular + Picks / Genre / Daily Mixes) is computed on-demand by + `PersonalizedPlaylistsService` — no persistence, every call reruns + the generator with `ORDER BY RANDOM()` so the result rotates. + +Post-overhaul (this module's responsibility): +- ALL personalized playlists land in a unified storage layer with a + stable (profile_id, kind, variant) identity, JSON config per + playlist (limit, diversity caps, popularity / recency filters, + exclude-recent-days, randomization seed), and a persistent track + list that only mutates when the playlist is explicitly refreshed. + +Tables created here: + +- ``personalized_playlists`` — one row per (profile, kind, variant). + Variants disambiguate kinds with multiple instances: + * ``time_machine``: variant = ``'1980s'`` / ``'1990s'`` / ... + * ``seasonal_mix``: variant = ``'halloween'`` / ``'christmas'`` / ... + * ``genre_playlist``: variant = ``'rock'`` / ``'electronic_dance'`` / ... + * ``daily_mix``: variant = ``'1'`` / ``'2'`` / ``'3'`` / ``'4'`` + * Singletons (``hidden_gems``, ``discovery_shuffle``, + ``popular_picks``, ``fresh_tape``, ``archives``): variant = ``''``. + Variant '' (empty) is used instead of NULL so the UNIQUE + constraint behaves predictably (NULL doesn't collide with NULL in + SQLite UNIQUE indexes — would let multiple singleton rows + coexist). + +- ``personalized_playlist_tracks`` — current snapshot per playlist. + Cleared + repopulated on refresh; never partial-mutates. + +- ``personalized_track_history`` — append-only log of which tracks + were served by which (profile, kind) over time. Powers the + ``exclude_recent_days`` config option so generators can avoid + recommending the same track twice in N days. + +The schema is created idempotently — `ensure_personalized_schema` +runs CREATE TABLE IF NOT EXISTS at startup, so existing installs +upgrade silently.""" + +from __future__ import annotations + +from typing import Any + +from utils.logging_config import get_logger + +logger = get_logger("database.personalized_schema") + + +PERSONALIZED_PLAYLISTS_DDL = """ +CREATE TABLE IF NOT EXISTS personalized_playlists ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + profile_id INTEGER NOT NULL DEFAULT 1, + kind TEXT NOT NULL, + variant TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL, + config_json TEXT NOT NULL DEFAULT '{}', + track_count INTEGER NOT NULL DEFAULT 0, + last_generated_at TIMESTAMP, + last_synced_at TIMESTAMP, + last_generation_source TEXT, + last_generation_error TEXT, + is_stale INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE (profile_id, kind, variant) +) +""" + +# Migration for installs that created the table before is_stale existed. +PERSONALIZED_PLAYLISTS_STALE_MIGRATION = """ +ALTER TABLE personalized_playlists ADD COLUMN is_stale INTEGER NOT NULL DEFAULT 0 +""" + +PERSONALIZED_PLAYLIST_TRACKS_DDL = """ +CREATE TABLE IF NOT EXISTS personalized_playlist_tracks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + playlist_id INTEGER NOT NULL, + position INTEGER NOT NULL, + spotify_track_id TEXT, + itunes_track_id TEXT, + deezer_track_id TEXT, + track_name TEXT, + artist_name TEXT, + album_name TEXT, + album_cover_url TEXT, + duration_ms INTEGER, + popularity INTEGER, + track_data_json TEXT, + FOREIGN KEY (playlist_id) REFERENCES personalized_playlists(id) ON DELETE CASCADE, + UNIQUE (playlist_id, position) +) +""" + +PERSONALIZED_PLAYLIST_TRACKS_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_personalized_tracks_playlist + ON personalized_playlist_tracks(playlist_id) +""" + +PERSONALIZED_TRACK_HISTORY_DDL = """ +CREATE TABLE IF NOT EXISTS personalized_track_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + profile_id INTEGER NOT NULL DEFAULT 1, + kind TEXT NOT NULL, + track_id TEXT NOT NULL, + served_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +) +""" + +PERSONALIZED_TRACK_HISTORY_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_personalized_history_lookup + ON personalized_track_history(profile_id, kind, track_id, served_at) +""" + + +def ensure_personalized_schema(connection: Any) -> None: + """Create the personalized-playlist tables + indexes if missing. + + Idempotent. Safe to call on every app startup. Caller is responsible + for committing the connection (we leave that to the caller so this + composes with other schema-init steps in one transaction). + """ + cursor = connection.cursor() + cursor.execute(PERSONALIZED_PLAYLISTS_DDL) + cursor.execute(PERSONALIZED_PLAYLIST_TRACKS_DDL) + cursor.execute(PERSONALIZED_PLAYLIST_TRACKS_INDEX) + cursor.execute(PERSONALIZED_TRACK_HISTORY_DDL) + cursor.execute(PERSONALIZED_TRACK_HISTORY_INDEX) + + # Add is_stale column on installs that created the table before + # this column existed. SQLite has no `ADD COLUMN IF NOT EXISTS` so + # we probe with PRAGMA + tolerate the OperationalError that fires + # when the column is already there. + cursor.execute("PRAGMA table_info(personalized_playlists)") + cols = {row[1] for row in cursor.fetchall()} + if 'is_stale' not in cols: + try: + cursor.execute(PERSONALIZED_PLAYLISTS_STALE_MIGRATION) + logger.info("Added is_stale column to personalized_playlists") + except Exception as e: + logger.debug("is_stale column migration: %s", e) + + logger.debug("Personalized-playlist schema ensured") + + +__all__ = [ + 'ensure_personalized_schema', + 'PERSONALIZED_PLAYLISTS_DDL', + 'PERSONALIZED_PLAYLIST_TRACKS_DDL', + 'PERSONALIZED_PLAYLIST_TRACKS_INDEX', + 'PERSONALIZED_TRACK_HISTORY_DDL', + 'PERSONALIZED_TRACK_HISTORY_INDEX', +] diff --git a/docker-compose.yml b/docker-compose.yml index a2d8a0d4..fe5ed945 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,26 @@ services: # - /path/to/music-library:/host/music:rw # Optional: Mount your music library for Plex/Jellyfin/Navidrome access # - /path/to/your/music:/music:ro + # + # ─── Torrent / Usenet download sources (Prowlarr integration) ──────── + # If you use the Torrent Only or Usenet Only download sources, SoulSync + # needs to read the files your torrent / usenet client downloads. + # + # EASIEST SETUP: point qBittorrent, SABnzbd / NZBGet, AND slskd at the + # SAME download folder. The ./downloads mount above is reused; nothing + # new to add here. Just configure each client's download path to + # /downloads (inside its own container) — files land in the same place + # SoulSync already reads from. + # + # SEPARATE FOLDERS (if you prefer): uncomment + edit these to point at + # the client's download folder. The in-container path must MATCH the + # save_path the client reports. + # - /path/to/qbittorrent/downloads:/downloads/torrents:rw + # - /path/to/sabnzbd/complete:/downloads/usenet:rw + # Then set the torrent client's save path to /downloads/torrents and + # SAB / NZBGet's complete folder to /downloads/usenet (or whatever + # in-container path you chose). Skip these if your downloader runs + # outside Docker and writes to a path already mounted above. extra_hosts: # Allow container to reach host services - "host.docker.internal:host-gateway" diff --git a/entrypoint.sh b/entrypoint.sh index 7d830525..8cde8e0f 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -40,7 +40,7 @@ if [ "$CURRENT_UID" != "$PUID" ] || [ "$CURRENT_GID" != "$PGID" ]; then DATA_OWNER=$(stat -c '%u:%g' /app/data 2>/dev/null || echo "unknown") if [ "$DATA_OWNER" != "$PUID:$PGID" ]; then echo "🔒 Fixing permissions on app directories..." - chown -R soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging 2>/dev/null || true + chown -R soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream 2>/dev/null || true else echo "✅ App directory permissions already correct" fi @@ -81,15 +81,15 @@ chown soulsync:soulsync /app/config/settings.py 2>/dev/null || true # Pre-mid-2026 the chown line had `|| true` but mkdir didn't — combined # with `set -e`, a permission-denied mkdir crashed the container into a # restart loop. Both lines are now best-effort. -mkdir -p /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging 2>/dev/null || true -chown soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging 2>/dev/null || true +mkdir -p /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream 2>/dev/null || true +chown soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream 2>/dev/null || true # Writability audit — surface a loud warning if any bind-mounted dir # isn't writable by the soulsync user. The restart-loop fix above makes # the container start regardless, but a non-writable Staging / downloads # / Transfer will fail silently inside the app (auto-import quarantine, # download writes). Better to log now than to debug missing files later. -for dir in /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/MusicVideos /app/scripts; do +for dir in /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream /app/MusicVideos /app/scripts; do if [ -d "$dir" ] && ! gosu soulsync test -w "$dir" 2>/dev/null; then echo "⚠️ WARNING: $dir is not writable by soulsync (uid $(id -u soulsync))." echo " Host bind-mount perms likely mismatch the PUID/PGID env vars." diff --git a/pr_description.md b/pr_description.md new file mode 100644 index 00000000..83fda10a --- /dev/null +++ b/pr_description.md @@ -0,0 +1,76 @@ +## Summary + +Adds torrent and usenet as release-oriented download sources backed by Prowlarr and the configured downloader clients. These sources can download full releases, stage the resulting audio files, and let SoulSync match/import the requested tracks through the existing post-processing pipeline. + +This PR also improves the torrent/usenet user experience with clearer release-download progress, source-aware service labels, library history provenance, and safer staged-release matching for album files that include featured-artist or bonus-track filename noise. + +## Scope + +- Adds torrent and usenet plugin support for release downloads. +- Adds album-bundle staging for single-source torrent/usenet album downloads. +- Keeps hybrid album downloads on per-track-capable sources by excluding torrent/usenet from hybrid album per-track searches. +- Allows torrent/usenet in hybrid for non-album track downloads, such as playlist singles and wishlist tracks. +- Selects the requested audio file from completed release folders instead of importing the first audio file blindly. +- Reports live album-bundle progress to the Downloads page and download modal. +- Records torrent/usenet provenance in library history. +- Adds UI polish for release-first download states. + +## Behavior Gates + +- Users who do not select torrent or usenet as a download source should not hit the new download paths. +- Single-source `torrent` / `usenet` album downloads use the release staging flow. +- Hybrid album downloads skip torrent/usenet and continue through the existing per-track source chain. +- Hybrid non-album downloads may use torrent/usenet when they are included in the hybrid order. + +## Notable Implementation Details + +- Torrent/usenet release downloads use private per-batch staging folders under the configured album-bundle staging root. +- Post-processing receives the real source label (`torrent` / `usenet`) so library history no longer falls back to `Soulseek`. +- Staging matching strips only conservative noise like `(feat. Artist)` and `(Bonus Track)` while preserving meaningful version text like `remix`, `extended`, `live`, and `acoustic`. +- When a staged release does not contain a requested track, the task is marked not found instead of repeatedly searching/downloading the same release. +- Completed release downloads expose all discovered audio files so post-processing can choose the best matching track. + +## Testing + +Manually tested: + +- Torrent-only album download for `good kid, m.A.A.d city (Deluxe)`. +- Torrent playlist/single-track flow. +- Album-bundle progress in the download modal and Downloads page. +- Torrent source labeling in library history for new imports. +- Staging match behavior for featured-artist filenames and bonus-track labels. +- Quarantine behavior for wrong-version matches. + +Automated tests run during development: + +```bash +.venv/bin/python -m pytest \ + tests/downloads/test_downloads_status.py \ + tests/test_album_bundle_dispatch.py \ + tests/downloads/test_downloads_staging.py \ + tests/test_torrent_usenet_plugins.py +``` + +```bash +.venv/bin/python -m pytest \ + tests/downloads/test_downloads_validation.py \ + tests/test_manual_pick_no_auto_retry.py \ + tests/downloads/test_downloads_post_processing.py \ + tests/downloads/test_downloads_task_worker.py \ + tests/imports/test_import_side_effects.py +``` + +Focused checks also passed for: + +- staged release feature suffix matching +- bonus-track title matching +- wrong-version separation +- private torrent album staging miss handling +- torrent/usenet history source labels + +## Reviewer Notes + +- This is intentionally gated behind torrent/usenet source selection, but it is still a new release-oriented download path and should be considered beta/experimental for first release. +- Remote downloader setups need SoulSync to be able to read the downloader save path. Local/all-in-one setups should be the easiest path. +- Existing library history rows that were previously recorded as `Soulseek` are not backfilled by this PR. +- Release matching is naturally fuzzier than track-native sources, so reviewer focus should stay on false positives, version handling, and staged-file selection. diff --git a/tests/automation/test_automation_blocks.py b/tests/automation/test_automation_blocks.py index 64d95cfa..151725bb 100644 --- a/tests/automation/test_automation_blocks.py +++ b/tests/automation/test_automation_blocks.py @@ -34,7 +34,8 @@ def _shape_check(items, allowed_types): _FIELD_TYPES = { 'number', 'select', 'time', 'multi_select', 'checkbox', 'text', - 'mirrored_playlist_select', 'signal_input', 'script_select', + 'mirrored_playlist_select', 'personalized_playlist_select', + 'signal_input', 'script_select', } diff --git a/tests/automation/test_handler_registration.py b/tests/automation/test_handler_registration.py new file mode 100644 index 00000000..b6660541 --- /dev/null +++ b/tests/automation/test_handler_registration.py @@ -0,0 +1,391 @@ +"""Engine-boundary tests for the automation handler registration. + +Per-handler boundary tests in the sibling test files prove each +handler's body works in isolation. These tests prove the +**registration layer** wires every handler to the right action name, +attaches the right guard, and registers the four progress callbacks +in the slots the engine expects. + +The kettui standard for refactor PRs: don't ship a "behavior +preserved" claim that's only validated at the function boundary. +Wire the seam — engine + register_all + deps — and exercise it. + +These tests use a minimal recording engine that captures every +``register_action_handler`` / ``register_progress_callbacks`` call, +plus a no-op ``add_scan_completion_callback`` on a fake scan +manager. No real AutomationEngine, no real DB, no real Flask app. +""" + +from __future__ import annotations + +import threading +from typing import Any, Dict, List, Tuple + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers import register_all + + +# Every action name `register_all` is expected to register. Drift +# (rename / new handler / removed handler) fails this test +# immediately so refactor PRs can't quietly drop a handler. +EXPECTED_ACTION_NAMES = frozenset({ + 'process_wishlist', + 'scan_watchlist', + 'scan_library', + 'refresh_mirrored', + 'sync_playlist', + 'discover_playlist', + 'playlist_pipeline', + 'personalized_pipeline', + 'start_database_update', + 'deep_scan_library', + 'run_duplicate_cleaner', + 'clear_quarantine', + 'cleanup_wishlist', + 'update_discovery_pool', + 'start_quality_scan', + 'backup_database', + 'refresh_beatport_cache', + 'clean_search_history', + 'clean_completed_downloads', + 'full_cleanup', + 'run_script', + 'search_and_download', +}) + +# Action names that MUST register a guard (duplicate-run prevention). +EXPECTED_GUARDED_ACTIONS = frozenset({ + 'process_wishlist', + 'scan_watchlist', + 'scan_library', + 'playlist_pipeline', + 'personalized_pipeline', + 'start_database_update', + 'deep_scan_library', + 'run_duplicate_cleaner', + 'start_quality_scan', +}) + + +class _RecordingEngine: + """Minimal AutomationEngine stand-in. Captures everything + register_all does so tests can assert on it.""" + + def __init__(self): + self.action_handlers: Dict[str, Dict[str, Any]] = {} + self.progress_callbacks: Tuple = () + self.emits: List[Tuple[str, dict]] = [] + + def register_action_handler(self, action_type, handler_fn, guard_fn=None): + self.action_handlers[action_type] = {'handler': handler_fn, 'guard': guard_fn} + + def register_progress_callbacks(self, init_fn, finish_fn, update_fn=None, history_fn=None): + self.progress_callbacks = (init_fn, finish_fn, update_fn, history_fn) + + def emit(self, event_name, payload): + self.emits.append((event_name, payload)) + + +class _RecordingScanMgr: + _current_server_type = 'plex' + + def __init__(self): + self.callbacks: List = [] + + def add_scan_completion_callback(self, cb): + self.callbacks.append(cb) + + +def _build_deps(engine, scan_mgr=None) -> AutomationDeps: + class _Logger: + def debug(self, *a, **k): pass + def info(self, *a, **k): pass + def warning(self, *a, **k): pass + def error(self, *a, **k): pass + + class _Cfg: + def get(self, key, default=None): return default + def get_active_media_server(self): return 'plex' + + return AutomationDeps( + engine=engine, + state=AutomationState(), + config_manager=_Cfg(), + update_progress=lambda *a, **k: None, + logger=_Logger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=scan_mgr, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + + +# ─── action handler registration ───────────────────────────────────── + + +class TestActionHandlerRegistration: + def test_every_expected_action_name_registered(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + registered = set(engine.action_handlers.keys()) + missing = EXPECTED_ACTION_NAMES - registered + extra = registered - EXPECTED_ACTION_NAMES + assert not missing, f"register_all dropped: {missing}" + assert not extra, f"register_all added unexpected: {extra}" + + def test_guarded_actions_have_a_guard(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + for name in EXPECTED_GUARDED_ACTIONS: + assert engine.action_handlers[name]['guard'] is not None, ( + f"action {name!r} expected to register a guard but didn't" + ) + + def test_unguarded_actions_have_no_guard(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + unguarded = EXPECTED_ACTION_NAMES - EXPECTED_GUARDED_ACTIONS + for name in unguarded: + assert engine.action_handlers[name]['guard'] is None, ( + f"action {name!r} unexpectedly registered a guard" + ) + + def test_every_handler_callable(self): + # Every registered handler must be callable with a config dict. + engine = _RecordingEngine() + register_all(_build_deps(engine)) + for name, entry in engine.action_handlers.items(): + handler = entry['handler'] + assert callable(handler), f"{name} handler is not callable" + + def test_every_guard_callable_returns_bool(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + for name, entry in engine.action_handlers.items(): + guard = entry['guard'] + if guard is None: + continue + value = guard() + assert isinstance(value, bool), ( + f"{name} guard returned non-bool: {type(value).__name__}" + ) + + +# ─── progress callback registration ────────────────────────────────── + + +class TestProgressCallbackRegistration: + def test_all_four_callbacks_registered(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + init_fn, finish_fn, update_fn, history_fn = engine.progress_callbacks + assert callable(init_fn) + assert callable(finish_fn) + assert callable(update_fn) + assert callable(history_fn) + + def test_progress_init_callback_invocable(self): + # Engine signature: init_fn(aid, name, action_type) + engine = _RecordingEngine() + captured: List[Tuple] = [] + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'init_automation_progress': lambda aid, name, at: captured.append((aid, name, at)), + }) + register_all(deps) + init_fn, _, _, _ = engine.progress_callbacks + init_fn('auto-1', 'My Auto', 'wishlist') + assert captured == [('auto-1', 'My Auto', 'wishlist')] + + def test_progress_finish_callback_invocable(self): + # Engine signature: finish_fn(aid, result) + engine = _RecordingEngine() + captured: List[Tuple] = [] + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'update_progress': lambda aid, **kw: captured.append((aid, kw)), + }) + register_all(deps) + _, finish_fn, _, _ = engine.progress_callbacks + # Non-_manages_own_progress result triggers update_progress emit. + finish_fn('auto-1', {'status': 'completed'}) + assert len(captured) == 1 + assert captured[0][0] == 'auto-1' + assert captured[0][1]['status'] == 'finished' + + def test_progress_finish_skips_self_managed(self): + engine = _RecordingEngine() + captured: List[Tuple] = [] + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'update_progress': lambda aid, **kw: captured.append((aid, kw)), + }) + register_all(deps) + _, finish_fn, _, _ = engine.progress_callbacks + finish_fn('auto-1', {'_manages_own_progress': True, 'status': 'completed'}) + assert captured == [] + + def test_history_callback_invocable_with_db(self): + # Engine signature: history_fn(aid, result) + engine = _RecordingEngine() + captured: List[Tuple] = [] + db_obj = object() + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'get_database': lambda: db_obj, + 'record_progress_history': lambda aid, result, db: captured.append((aid, result, db)), + }) + register_all(deps) + _, _, _, history_fn = engine.progress_callbacks + history_fn('auto-1', {'status': 'completed'}) + assert captured == [('auto-1', {'status': 'completed'}, db_obj)] + + +# ─── library_scan_completed wiring ─────────────────────────────────── + + +class TestLibraryScanCompletedEmitter: + def test_no_scan_manager_safe(self): + # Should not raise when scan manager is absent (test/headless mode). + engine = _RecordingEngine() + register_all(_build_deps(engine, scan_mgr=None)) + # No callbacks captured (no scan manager to register against), + # but engine still has all the handlers. + assert engine.action_handlers + + def test_scan_completion_callback_registered(self): + engine = _RecordingEngine() + scan_mgr = _RecordingScanMgr() + register_all(_build_deps(engine, scan_mgr=scan_mgr)) + assert len(scan_mgr.callbacks) == 1 + # Invoking the callback should fire engine.emit('library_scan_completed', ...). + scan_mgr.callbacks[0]() + assert engine.emits == [('library_scan_completed', {'server_type': 'plex'})] + + +# ─── handler invocation through the engine boundary ───────────────── + + +class TestHandlerInvocation: + """For each registered handler, exercise the lambda the engine + would call. Verifies the deps closure is captured correctly + the + handler returns a result dict. + + Forces every long-running handler down its guard short-circuit + path by pre-setting the relevant `*_state` dicts to ``running`` + (database_update, deep_scan, duplicate_cleaner, quality_scanner) + or by pre-occupying the state flags (scan_library, playlist_pipeline). + Other handlers either return error early (no playlist specified, + no scan manager, etc) or run cleanly against the no-op stub deps. + """ + + def test_every_handler_returns_dict(self): + engine = _RecordingEngine() + # Pre-set state dicts so guarded handlers skip-return cleanly. + running_state = {'status': 'running'} + active_batches = {'b1': {'phase': 'downloading'}} + + # Stub DB that satisfies the handlers reaching for it on + # short paths. cleanup_wishlist calls remove_wishlist_duplicates. + # update_discovery_pool's exception path swallows missing + # scanner. Other handlers either short-circuit or use stub + # callables. + class _StubDB: + def remove_wishlist_duplicates(self, profile_id): return 0 + def get_mirrored_playlists(self): return [] + def get_mirrored_playlist(self, _id): return None + def get_mirrored_playlist_tracks(self, _id): return [] + + # Stub Flask app so refresh_beatport_cache's test_client call + # doesn't crash. Use a minimal context manager. + class _FakeResponse: + status_code = 200 + class _FakeClient: + def __enter__(self): return self + def __exit__(self, *a): return False + def get(self, _path): return _FakeResponse() + class _StubApp: + def test_client(self): return _FakeClient() + + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'get_db_update_state': lambda: running_state, + 'get_duplicate_cleaner_state': lambda: running_state, + 'get_quality_scanner_state': lambda: running_state, + 'get_download_batches': lambda: active_batches, # forces clean_completed_downloads to skip + 'get_database': lambda: _StubDB(), + 'get_app': lambda: _StubApp(), + }) + # Pre-set state flags too so scan_library + playlist_pipeline guards fire. + deps.state.scan_library_automation_id = 'someone-else' + deps.state.pipeline_running = True + + # Patch time.sleep across handler modules so refresh_beatport_cache + # (which sleeps 2s between sections) doesn't extend the test. + import core.automation.handlers.maintenance as maint_mod + original_sleep = maint_mod.time.sleep + maint_mod.time.sleep = lambda _: None + + register_all(deps) + + try: + for name, entry in engine.action_handlers.items(): + handler = entry['handler'] + # Minimal config: trigger the natural error/skip paths + # for handlers that need a playlist_id, query, script_name etc. + result = handler({'_automation_id': 'test', 'all': False}) + assert isinstance(result, dict), ( + f"handler {name!r} returned {type(result).__name__}, expected dict" + ) + assert 'status' in result, ( + f"handler {name!r} returned dict without 'status': {list(result.keys())}" + ) + finally: + maint_mod.time.sleep = original_sleep diff --git a/tests/automation/test_handlers_maintenance.py b/tests/automation/test_handlers_maintenance.py new file mode 100644 index 00000000..a3844b77 --- /dev/null +++ b/tests/automation/test_handlers_maintenance.py @@ -0,0 +1,393 @@ +"""Boundary tests for the maintenance + misc automation handlers +(database_update / deep_scan_library / duplicate_cleaner / +quality_scanner / clear_quarantine / cleanup_wishlist / +update_discovery_pool / backup_database / refresh_beatport_cache / +clean_search_history / clean_completed_downloads / full_cleanup / +run_script / search_and_download). + +Each handler is tested as a pure function via stub deps. The bodies +are mechanical lifts of the closures that used to live in +``web_server._register_automation_handlers`` — these tests pin the +seam (deps wiring, exception swallow contract, return shapes, +guard short-circuits) so future drift fails here, not at runtime +against real executors / clients.""" + +from __future__ import annotations + +import os +import threading +from typing import Any, Dict, List + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.database_update import ( + auto_start_database_update, auto_deep_scan_library, +) +from core.automation.handlers.duplicate_cleaner import auto_run_duplicate_cleaner +from core.automation.handlers.quality_scanner import auto_start_quality_scan +from core.automation.handlers.maintenance import ( + auto_clear_quarantine, auto_cleanup_wishlist, + auto_update_discovery_pool, auto_backup_database, +) +from core.automation.handlers.download_cleanup import ( + auto_clean_search_history, auto_clean_completed_downloads, +) +from core.automation.handlers.run_script import auto_run_script +from core.automation.handlers.search_and_download import auto_search_and_download + + +class _StubLogger: + def debug(self, *a, **k): pass + def info(self, *a, **k): pass + def warning(self, *a, **k): pass + def error(self, *a, **k): pass + + +class _StubConfig: + def __init__(self, values=None): + self._values = values or {} + + def get(self, key, default=None): + return self._values.get(key, default) + + def get_active_media_server(self): + return 'plex' + + +def _build_deps(**overrides) -> AutomationDeps: + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=_StubConfig(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── database_update / deep_scan ────────────────────────────────────── + + +class _StubExecutor: + def __init__(self): + self.submits: List[tuple] = [] + + def submit(self, fn, *args, **kwargs): + self.submits.append((fn, args, kwargs)) + + +class TestDatabaseUpdate: + def test_already_running_returns_skipped(self): + state = {'status': 'running'} + deps = _build_deps(get_db_update_state=lambda: state) + result = auto_start_database_update({}, deps) + assert result == {'status': 'skipped', 'reason': 'Database update already running'} + + def test_set_db_update_automation_id_called(self): + # Handler must propagate the automation id through the deps + # setter so the legacy global stays in sync. + captured: List[Any] = [] + state = {'status': 'idle'} + # Make the polling loop terminate immediately by flipping + # the status as soon as it's set to 'running'. + executor = _StubExecutor() + + def fake_task(*_a, **_k): + state['status'] = 'finished' + + executor.submit = lambda fn, *a, **k: fake_task() + deps = _build_deps( + get_db_update_state=lambda: state, + db_update_executor=executor, + set_db_update_automation_id=lambda v: captured.append(v), + run_db_update_task=fake_task, + ) + # Replace time.sleep so we don't actually wait + import core.automation.handlers.database_update as module + original = module.time.sleep + module.time.sleep = lambda _: None + try: + result = auto_start_database_update({'_automation_id': 'auto-1'}, deps) + finally: + module.time.sleep = original + assert captured == ['auto-1'] + assert result['status'] == 'completed' + + +class TestDeepScan: + def test_already_running_returns_skipped(self): + state = {'status': 'running'} + deps = _build_deps(get_db_update_state=lambda: state) + result = auto_deep_scan_library({}, deps) + assert result == {'status': 'skipped', 'reason': 'Database update already running'} + + +# ─── duplicate_cleaner ──────────────────────────────────────────────── + + +class TestDuplicateCleaner: + def test_already_running_returns_skipped(self): + state = {'status': 'running'} + deps = _build_deps(get_duplicate_cleaner_state=lambda: state) + result = auto_run_duplicate_cleaner({}, deps) + assert result == {'status': 'skipped', 'reason': 'Duplicate cleaner already running'} + + +# ─── quality_scanner ────────────────────────────────────────────────── + + +class TestQualityScanner: + def test_already_running_returns_skipped(self): + state = {'status': 'running'} + deps = _build_deps(get_quality_scanner_state=lambda: state) + result = auto_start_quality_scan({}, deps) + assert result == {'status': 'skipped', 'reason': 'Quality scan already running'} + + +# ─── clear_quarantine ──────────────────────────────────────────────── + + +class TestClearQuarantine: + def test_no_quarantine_folder_returns_zero(self, tmp_path): + # Point at a non-existent path. + deps = _build_deps( + config_manager=_StubConfig({'soulseek.download_path': str(tmp_path / 'nonexistent')}), + docker_resolve_path=lambda p: p, + ) + result = auto_clear_quarantine({}, deps) + assert result == {'status': 'completed', 'removed': '0'} + + def test_clears_files_from_quarantine(self, tmp_path): + download_path = tmp_path + quarantine_path = download_path / 'ss_quarantine' + quarantine_path.mkdir() + (quarantine_path / 'a.flac').write_bytes(b'') + (quarantine_path / 'b.flac').write_bytes(b'') + deps = _build_deps( + config_manager=_StubConfig({'soulseek.download_path': str(download_path)}), + docker_resolve_path=lambda p: p, + ) + result = auto_clear_quarantine({}, deps) + assert result == {'status': 'completed', 'removed': '2'} + + +# ─── cleanup_wishlist ──────────────────────────────────────────────── + + +class TestCleanupWishlist: + def test_returns_count_from_db(self): + class _DB: + def remove_wishlist_duplicates(self, profile_id): + assert profile_id == 1 + return 7 + + deps = _build_deps(get_database=lambda: _DB()) + result = auto_cleanup_wishlist({}, deps) + assert result == {'status': 'completed', 'removed': '7'} + + def test_returns_zero_when_db_returns_falsey(self): + class _DB: + def remove_wishlist_duplicates(self, profile_id): + return None + + deps = _build_deps(get_database=lambda: _DB()) + result = auto_cleanup_wishlist({}, deps) + assert result == {'status': 'completed', 'removed': '0'} + + +# ─── update_discovery_pool ─────────────────────────────────────────── + + +class TestUpdateDiscoveryPool: + def test_success(self): + called: List[Any] = [] + + class _Scanner: + def update_discovery_pool_incremental(self, profile_id): + called.append(profile_id) + + deps = _build_deps( + get_watchlist_scanner=lambda spc: _Scanner(), + get_current_profile_id=lambda: 7, + ) + result = auto_update_discovery_pool({}, deps) + assert result['status'] == 'completed' + assert result['_manages_own_progress'] is True + assert called == [7] + + def test_exception_swallowed(self): + def boom(_): + raise RuntimeError('scanner down') + + deps = _build_deps(get_watchlist_scanner=boom) + result = auto_update_discovery_pool({}, deps) + assert result['status'] == 'error' + assert 'scanner down' in result['reason'] + + +# ─── backup_database ───────────────────────────────────────────────── + + +class TestBackupDatabase: + def test_missing_database_returns_error(self, tmp_path, monkeypatch): + monkeypatch.setenv('DATABASE_PATH', str(tmp_path / 'no.db')) + deps = _build_deps() + result = auto_backup_database({}, deps) + assert result == {'status': 'error', 'reason': 'Database file not found'} + + +# ─── clean_search_history ──────────────────────────────────────────── + + +class TestCleanSearchHistory: + def test_soulseek_inactive_skips(self): + deps = _build_deps( + config_manager=_StubConfig({'download_source.mode': 'tidal'}), + ) + result = auto_clean_search_history({}, deps) + assert result == {'status': 'skipped'} + + def test_no_orchestrator_skips(self): + deps = _build_deps( + config_manager=_StubConfig({'download_source.mode': 'soulseek'}), + download_orchestrator=None, + ) + result = auto_clean_search_history({}, deps) + assert result == {'status': 'skipped'} + + +# ─── clean_completed_downloads ─────────────────────────────────────── + + +class TestCleanCompletedDownloads: + def test_active_batches_skip(self): + # Active batch present → handler returns 'completed' without doing anything. + deps = _build_deps( + get_download_batches=lambda: {'b1': {'phase': 'downloading'}}, + get_download_tasks=lambda: {}, + ) + result = auto_clean_completed_downloads({}, deps) + assert result == {'status': 'completed'} + + +# ─── run_script ────────────────────────────────────────────────────── + + +class TestRunScript: + def test_no_script_name_returns_error(self): + deps = _build_deps() + result = auto_run_script({}, deps) + assert result == {'status': 'error', 'error': 'No script selected'} + + def test_path_traversal_blocked(self, tmp_path): + scripts_dir = tmp_path / 'scripts' + scripts_dir.mkdir() + # Place a script OUTSIDE the scripts dir + try to reach it + # via ../ traversal. + evil = tmp_path / 'evil.sh' + evil.write_text('#!/bin/bash\necho evil') + deps = _build_deps( + config_manager=_StubConfig({'scripts.path': str(scripts_dir)}), + docker_resolve_path=lambda p: p, + ) + result = auto_run_script({'script_name': '../evil.sh'}, deps) + assert result['status'] == 'error' + assert 'path traversal' in result['error'].lower() + + def test_missing_script_returns_error(self, tmp_path): + scripts_dir = tmp_path / 'scripts' + scripts_dir.mkdir() + deps = _build_deps( + config_manager=_StubConfig({'scripts.path': str(scripts_dir)}), + ) + result = auto_run_script({'script_name': 'no_such_script.sh'}, deps) + assert result['status'] == 'error' + assert 'not found' in result['error'].lower() + + +# ─── search_and_download ───────────────────────────────────────────── + + +class TestSearchAndDownload: + def test_no_query_returns_error(self): + deps = _build_deps() + result = auto_search_and_download({}, deps) + assert result == {'status': 'error', 'error': 'No search query provided'} + + def test_query_from_event_data_used(self): + captured_queries: List[str] = [] + + class _Orchestrator: + async def search_and_download_best(self, q): + captured_queries.append(q) + return 'dl-id-123' + + deps = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda coro: 'dl-id-123', + ) + result = auto_search_and_download( + {'_event_data': {'query': 'Adele Hello'}}, deps, + ) + assert result['status'] == 'completed' + assert result['query'] == 'Adele Hello' + assert result['download_id'] == 'dl-id-123' + + def test_no_match_returns_not_found(self): + class _Orchestrator: + async def search_and_download_best(self, q): + return None + + deps = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda coro: None, + ) + result = auto_search_and_download({'query': 'xyz'}, deps) + assert result['status'] == 'not_found' + assert result['query'] == 'xyz' diff --git a/tests/automation/test_handlers_personalized_pipeline.py b/tests/automation/test_handlers_personalized_pipeline.py new file mode 100644 index 00000000..049cd466 --- /dev/null +++ b/tests/automation/test_handlers_personalized_pipeline.py @@ -0,0 +1,529 @@ +"""Boundary tests for the personalized playlist pipeline handler. + +Pin every shape: empty kinds error, refresh_first behaviour, snapshot +load + sync dispatch, missing-tracks skip, exception swallowing, +pipeline_running flag cleanup, sync payload shape passed to +_run_sync_task.""" + +from __future__ import annotations + +import threading +from types import SimpleNamespace +from typing import Any, List + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.personalized_pipeline import ( + auto_personalized_pipeline, + _build_payloads_for_kinds, + _track_to_sync_shape, + _sync_personalized_playlist, +) + + +class _StubLogger: + def debug(self, *a, **k): pass + def info(self, *a, **k): pass + def warning(self, *a, **k): pass + def error(self, *a, **k): pass + + +def _build_deps(**overrides) -> AutomationDeps: + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=object(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── Track shape converter ─────────────────────────────────────────── + + +class TestTrackToSyncShape: + def test_basic_shape(self): + track = SimpleNamespace( + track_name='Song', artist_name='Artist', album_name='Album', + spotify_track_id='sp-1', itunes_track_id=None, deezer_track_id=None, + duration_ms=200000, + ) + out = _track_to_sync_shape(track) + assert out == { + 'name': 'Song', + 'artists': [{'name': 'Artist'}], + 'album': {'name': 'Album'}, + 'duration_ms': 200000, + 'id': 'sp-1', + } + + def test_falls_back_through_source_ids(self): + t1 = SimpleNamespace(track_name='', artist_name='', album_name='', + spotify_track_id=None, itunes_track_id='it-1', + deezer_track_id=None, duration_ms=0) + assert _track_to_sync_shape(t1)['id'] == 'it-1' + + t2 = SimpleNamespace(track_name='', artist_name='', album_name='', + spotify_track_id=None, itunes_track_id=None, + deezer_track_id='dz-1', duration_ms=0) + assert _track_to_sync_shape(t2)['id'] == 'dz-1' + + def test_no_id_returns_empty_string(self): + t = SimpleNamespace(track_name='X', artist_name='Y', album_name='Z', + spotify_track_id=None, itunes_track_id=None, + deezer_track_id=None, duration_ms=0) + assert _track_to_sync_shape(t)['id'] == '' + + def test_preserves_enriched_track_data_for_wishlist_metadata(self): + track = SimpleNamespace( + track_name='Bare Name', artist_name='Bare Artist', album_name='Bare Album', + spotify_track_id='sp-rich', itunes_track_id=None, deezer_track_id=None, + album_cover_url=None, duration_ms=200000, popularity=33, + track_data_json={ + 'id': 'sp-rich', + 'name': 'Rich Name', + 'artists': [{'name': 'Rich Artist', 'id': 'artist-1'}], + 'album': { + 'id': 'album-1', + 'name': 'Rich Album', + 'images': [{'url': 'https://example.test/cover.jpg'}], + }, + 'duration_ms': 201000, + 'preview_url': 'https://example.test/preview.mp3', + }, + ) + + out = _track_to_sync_shape(track) + + assert out['name'] == 'Rich Name' + assert out['artists'][0] == {'name': 'Rich Artist', 'id': 'artist-1'} + assert out['album']['id'] == 'album-1' + assert out['album']['images'][0]['url'] == 'https://example.test/cover.jpg' + assert out['preview_url'] == 'https://example.test/preview.mp3' + + def test_album_cover_url_fills_album_images_when_no_rich_blob(self): + track = SimpleNamespace( + track_name='Song', artist_name='Artist', album_name='Album', + spotify_track_id='sp-1', itunes_track_id=None, deezer_track_id=None, + album_cover_url='https://example.test/fallback.jpg', + duration_ms=200000, track_data_json=None, + ) + + out = _track_to_sync_shape(track) + + assert out['album'] == { + 'name': 'Album', + 'images': [{'url': 'https://example.test/fallback.jpg'}], + } + + +# ─── Empty / config validation ────────────────────────────────────── + + +class TestEmptyConfig: + def test_no_kinds_returns_error_and_clears_flag(self): + deps = _build_deps() + deps.state.set_pipeline_running(True) # simulate already-running + result = auto_personalized_pipeline({}, deps) + assert result['status'] == 'error' + assert 'No personalized playlist' in result['error'] + assert deps.state.pipeline_running is False + + def test_empty_kinds_list_returns_error(self): + deps = _build_deps() + result = auto_personalized_pipeline({'kinds': []}, deps) + assert result['status'] == 'error' + assert deps.state.pipeline_running is False + + def test_non_list_kinds_returns_error(self): + deps = _build_deps() + result = auto_personalized_pipeline({'kinds': 'not_a_list'}, deps) + assert result['status'] == 'error' + + +# ─── Payload building ─────────────────────────────────────────────── + + +class _StubManagerNoTracks: + def ensure_playlist(self, kind, variant, profile_id): + # last_generated_at non-None so pipeline treats the snapshot as + # already-generated-but-empty (rather than first-run-needs-gen). + return SimpleNamespace( + id=1, name=f'{kind}-{variant}', kind=kind, variant=variant, + is_stale=False, last_generated_at='2026-05-15T20:00:00', + ) + + def refresh_playlist(self, kind, variant, profile_id): + return self.ensure_playlist(kind, variant, profile_id) + + def get_playlist_tracks(self, playlist_id): + return [] + + +class _StubManagerWithTracks: + def __init__(self, tracks_per_kind=None): + self.tracks_per_kind = tracks_per_kind or {} + self.refresh_calls: List[tuple] = [] + self.ensure_calls: List[tuple] = [] + + def ensure_playlist(self, kind, variant, profile_id): + self.ensure_calls.append((kind, variant, profile_id)) + return SimpleNamespace( + id=hash((kind, variant)) % 10000, + name=f'{kind}-{variant or "S"}', kind=kind, variant=variant, + is_stale=False, last_generated_at='2026-05-15T20:00:00', + ) + + def refresh_playlist(self, kind, variant, profile_id): + self.refresh_calls.append((kind, variant, profile_id)) + # Mirror real manager: refresh returns a record without invoking + # the public ensure_playlist API path again. + return SimpleNamespace( + id=hash((kind, variant)) % 10000, + name=f'{kind}-{variant or "S"}', kind=kind, variant=variant, + is_stale=False, last_generated_at='2026-05-15T20:00:00', + ) + + def get_playlist_tracks(self, playlist_id): + # Return all tracks regardless of id — tests scope to one playlist at a time. + for tracks in self.tracks_per_kind.values(): + if tracks: + return [SimpleNamespace( + track_name=t['name'], artist_name=t.get('artist', 'A'), + album_name=t.get('album', 'Al'), + spotify_track_id=t.get('id'), + itunes_track_id=None, deezer_track_id=None, + duration_ms=200000, + ) for t in tracks] + return [] + + +class TestPayloadBuilding: + def test_skips_kinds_with_no_tracks(self): + deps = _build_deps() + manager = _StubManagerNoTracks() + payloads = _build_payloads_for_kinds( + deps, manager, + [{'kind': 'hidden_gems'}, {'kind': 'discovery_shuffle'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert payloads == [] + + def test_skips_invalid_entries(self): + deps = _build_deps() + manager = _StubManagerNoTracks() + payloads = _build_payloads_for_kinds( + deps, manager, + ['not-a-dict', {}, {'variant': 'no-kind'}], # all invalid + profile_id=1, automation_id=None, refresh_first=False, + ) + assert payloads == [] + + def test_refresh_first_calls_refresh(self): + deps = _build_deps() + manager = _StubManagerWithTracks( + tracks_per_kind={'hidden_gems': [{'name': 'T', 'id': 'sp-1'}]}, + ) + _build_payloads_for_kinds( + deps, manager, + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=True, + ) + assert manager.refresh_calls == [('hidden_gems', '', 1)] + assert manager.ensure_calls == [] + + def test_no_refresh_calls_ensure(self): + deps = _build_deps() + manager = _StubManagerWithTracks( + tracks_per_kind={'hidden_gems': [{'name': 'T', 'id': 'sp-1'}]}, + ) + _build_payloads_for_kinds( + deps, manager, + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert manager.ensure_calls == [('hidden_gems', '', 1)] + assert manager.refresh_calls == [] + + def test_payload_shape(self): + deps = _build_deps() + manager = _StubManagerWithTracks( + tracks_per_kind={'hidden_gems': [ + {'name': 'Track1', 'id': 'sp-1'}, + {'name': 'Track2', 'id': 'sp-2'}, + ]}, + ) + payloads = _build_payloads_for_kinds( + deps, manager, + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert len(payloads) == 1 + p = payloads[0] + assert p['kind'] == 'hidden_gems' + assert p['variant'] == '' + assert p['name'] == 'hidden_gems-S' + assert p['sync_id'].startswith('auto_personalized_hidden_gems_') + assert len(p['tracks_json']) == 2 + assert p['tracks_json'][0]['id'] == 'sp-1' + + def test_stale_snapshot_auto_refreshes_even_without_refresh_first(self): + """When the manager reports is_stale=True, the pipeline refreshes + regardless of the refresh_first config flag — the source data + (discovery_pool / curated lists) changed, so the snapshot must + be regenerated before syncing or we'd push stale data.""" + deps = _build_deps() + # Stub manager whose ensure_playlist returns a stale record. + # refresh_playlist should still get called. + refresh_called = [] + + class _StaleMgr: + def ensure_playlist(self, kind, variant, profile_id): + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, is_stale=True, + last_generated_at='2026-05-15T20:00:00', + ) + def refresh_playlist(self, kind, variant, profile_id): + refresh_called.append((kind, variant)) + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, is_stale=False, + last_generated_at='2026-05-15T20:00:00', + ) + def get_playlist_tracks(self, _id): + return [SimpleNamespace( + track_name='Refreshed', artist_name='A', album_name='Al', + spotify_track_id='sp-fresh', itunes_track_id=None, + deezer_track_id=None, duration_ms=200000, + )] + + payloads = _build_payloads_for_kinds( + deps, _StaleMgr(), + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert refresh_called == [('hidden_gems', '')] + assert len(payloads) == 1 + assert payloads[0]['tracks_json'][0]['name'] == 'Refreshed' + + def test_non_stale_snapshot_skips_refresh(self): + """When the snapshot is fresh AND refresh_first is False, just + read the existing tracks without re-running the generator.""" + deps = _build_deps() + refresh_called = [] + + class _FreshMgr: + def ensure_playlist(self, kind, variant, profile_id): + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, is_stale=False, + last_generated_at='2026-05-15T20:00:00', + ) + def refresh_playlist(self, *_a, **_k): + refresh_called.append('called') + return SimpleNamespace( + id=1, name='x', kind='x', variant='', is_stale=False, + last_generated_at='2026-05-15T20:00:00', + ) + def get_playlist_tracks(self, _id): + return [SimpleNamespace( + track_name='Cached', artist_name='A', album_name='Al', + spotify_track_id='sp-1', itunes_track_id=None, + deezer_track_id=None, duration_ms=200000, + )] + + _build_payloads_for_kinds( + deps, _FreshMgr(), + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert refresh_called == [] + + def test_never_generated_snapshot_triggers_first_refresh(self): + """First-run case: pipeline picks a brand-new kind, ensure_playlist + auto-creates the row with track_count=0 and last_generated_at=None. + Without this branch the pipeline would read the empty snapshot and + silently skip — user picked a kind and got nothing. With the branch, + last_generated_at=None forces a refresh so the generator actually runs.""" + deps = _build_deps() + refresh_called = [] + + class _NeverGenMgr: + def ensure_playlist(self, kind, variant, profile_id): + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, + is_stale=False, last_generated_at=None, + ) + def refresh_playlist(self, kind, variant, profile_id): + refresh_called.append((kind, variant)) + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, + is_stale=False, last_generated_at='2026-05-15T20:00:00', + ) + def get_playlist_tracks(self, _id): + return [SimpleNamespace( + track_name='Generated', artist_name='A', album_name='Al', + spotify_track_id='sp-new', itunes_track_id=None, + deezer_track_id=None, duration_ms=200000, + )] + + payloads = _build_payloads_for_kinds( + deps, _NeverGenMgr(), + [{'kind': 'fresh_tape'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert refresh_called == [('fresh_tape', '')] + assert len(payloads) == 1 + assert payloads[0]['tracks_json'][0]['name'] == 'Generated' + + def test_manager_exception_swallowed_continues_to_next(self): + deps = _build_deps() + + class _ExplodingMgr: + def __init__(self): + self.calls = [] + def ensure_playlist(self, kind, variant, profile_id): + self.calls.append(kind) + if kind == 'broken': + raise RuntimeError('manager boom') + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, is_stale=False, + last_generated_at='2026-05-15T20:00:00', + ) + def get_playlist_tracks(self, _id): + return [] + + mgr = _ExplodingMgr() + # broken raises, hidden_gems proceeds (just no tracks). + payloads = _build_payloads_for_kinds( + deps, mgr, + [{'kind': 'broken'}, {'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert mgr.calls == ['broken', 'hidden_gems'] + assert payloads == [] # neither produced tracks + + +# ─── Sync launch ──────────────────────────────────────────────────── + + +class TestSyncLaunch: + def test_sync_one_playlist_starts_thread(self): + captured: List[tuple] = [] + + def fake_run_sync_task(*args): + captured.append(args) + + deps = _build_deps( + run_sync_task=fake_run_sync_task, + get_current_profile_id=lambda: 7, + ) + payload = { + 'sync_id': 'auto_personalized_hidden_gems_', + 'name': 'Hidden Gems', + 'tracks_json': [{'name': 'X', 'id': 'sp-1'}], + 'image_url': '', + } + result = _sync_personalized_playlist(deps, payload) + assert result['status'] == 'started' + # Wait for thread to invoke fake_run_sync_task. + for _ in range(100): + if captured: + break + import time + time.sleep(0.01) + assert len(captured) == 1 + # Args: (sync_id, name, tracks_json, automation_id, profile_id, image_url) + assert captured[0][0] == 'auto_personalized_hidden_gems_' + assert captured[0][1] == 'Hidden Gems' + assert captured[0][3] is None # automation_id muted + assert captured[0][4] == 7 # profile_id + + +# ─── Full pipeline (with stubbed manager + sync states) ───────────── + + +class TestPipelineHappyPath: + def test_pipeline_completes_with_synced_count(self): + # Stub manager returns one playlist with 2 tracks. + manager = _StubManagerWithTracks( + tracks_per_kind={'hidden_gems': [ + {'name': 'A', 'id': 'sp-1'}, + {'name': 'B', 'id': 'sp-2'}, + ]}, + ) + + # sync_states populated as if the sync background task finished. + sync_states_storage = {} + + def fake_run_sync(sync_id, name, tracks, aid, pid, img): + sync_states_storage[sync_id] = { + 'status': 'finished', + 'result': {'matched_tracks': 2}, + } + + deps = _build_deps( + build_personalized_manager=lambda: manager, + run_sync_task=fake_run_sync, + get_sync_states=lambda: sync_states_storage, + ) + # Patch time.sleep in shared helper so test doesn't take 2s per iter. + import core.automation.handlers._pipeline_shared as shared + orig = shared.time.sleep + shared.time.sleep = lambda _: None + try: + result = auto_personalized_pipeline( + {'_automation_id': 'auto-1', 'kinds': [{'kind': 'hidden_gems'}]}, + deps, + ) + finally: + shared.time.sleep = orig + assert result['status'] == 'completed' + assert result['_manages_own_progress'] is True + # Pipeline-running flag cleaned up. + assert deps.state.pipeline_running is False diff --git a/tests/automation/test_handlers_playlist.py b/tests/automation/test_handlers_playlist.py new file mode 100644 index 00000000..1a1548ef --- /dev/null +++ b/tests/automation/test_handlers_playlist.py @@ -0,0 +1,400 @@ +"""Boundary tests for the playlist-lifecycle automation handlers +(``refresh_mirrored``, ``sync_playlist``, ``discover_playlist``, +``playlist_pipeline``). + +The handlers themselves are mechanical lifts of the closures that +used to live in ``web_server._register_automation_handlers`` — these +tests pin the seam so the wiring stays correct (deps are read from +the deps object, not module-level globals; cross-handler calls in +the pipeline still compose; failure paths still return clear status +shapes). + +Source-specific branches inside ``refresh_mirrored`` (Spotify auth ++ public-embed fallback, Deezer / Tidal / YouTube) are validated +end-to-end via fake clients in ``deps`` rather than per-source +because they're a verbatim lift — drift would show up here as a +behavior change.""" + +from __future__ import annotations + +import json +import threading +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.discover_playlist import auto_discover_playlist +from core.automation.handlers.playlist_pipeline import auto_playlist_pipeline +from core.automation.handlers.refresh_mirrored import auto_refresh_mirrored +from core.automation.handlers.sync_playlist import auto_sync_playlist + + +# ─── shared scaffolding ────────────────────────────────────────────── + + +class _StubLogger: + def __init__(self): + self.messages: List[tuple] = [] + + def debug(self, *a, **k): self.messages.append(('debug', a)) + def info(self, *a, **k): self.messages.append(('info', a)) + def warning(self, *a, **k): self.messages.append(('warning', a)) + def error(self, *a, **k): self.messages.append(('error', a)) + + +@dataclass +class _StubDB: + """Fake MusicDatabase — minimal surface used by the playlist handlers.""" + + playlists: List[dict] = field(default_factory=list) + playlist_tracks: Dict[int, List[dict]] = field(default_factory=dict) + extra_data_maps: Dict[int, Dict[str, str]] = field(default_factory=dict) + mirror_calls: List[dict] = field(default_factory=list) + + def get_mirrored_playlists(self) -> list: + return list(self.playlists) + + def get_mirrored_playlist(self, playlist_id: int) -> Optional[dict]: + for p in self.playlists: + if int(p.get('id', -1)) == int(playlist_id): + return p + return None + + def get_mirrored_playlist_tracks(self, playlist_id: int) -> list: + return list(self.playlist_tracks.get(int(playlist_id), [])) + + def get_mirrored_tracks_extra_data_map(self, playlist_id: int) -> dict: + return dict(self.extra_data_maps.get(int(playlist_id), {})) + + def mirror_playlist(self, **kwargs) -> None: + self.mirror_calls.append(kwargs) + + +def _build_deps(**overrides) -> AutomationDeps: + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=object(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: _StubDB(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── discover_playlist ─────────────────────────────────────────────── + + +class TestDiscoverPlaylist: + def test_no_playlist_id_returns_error(self): + deps = _build_deps() + result = auto_discover_playlist({}, deps) + assert result == {'status': 'error', 'reason': 'No playlist specified'} + + def test_specific_playlist_id_starts_worker(self): + db = _StubDB(playlists=[{'id': 42, 'name': 'Test Playlist'}]) + called: List[Any] = [] + deps = _build_deps( + get_database=lambda: db, + run_playlist_discovery_worker=lambda *a, **k: called.append((a, k)), + ) + result = auto_discover_playlist({'playlist_id': '42', '_automation_id': 'auto-1'}, deps) + assert result['status'] == 'started' + assert result['_manages_own_progress'] is True + assert result['playlist_count'] == '1' + # Worker spawned on a thread; give it a moment. + for _ in range(50): + if called: + break + import time + time.sleep(0.01) + assert len(called) == 1 + + def test_all_playlists_includes_every_one(self): + db = _StubDB(playlists=[ + {'id': 1, 'name': 'A'}, {'id': 2, 'name': 'B'}, {'id': 3, 'name': 'C'}, + ]) + deps = _build_deps(get_database=lambda: db) + result = auto_discover_playlist({'all': True}, deps) + assert result['playlist_count'] == '3' + assert 'A' in result['playlists'] + assert 'B' in result['playlists'] + assert 'C' in result['playlists'] + + def test_no_playlists_in_db_returns_error(self): + deps = _build_deps(get_database=lambda: _StubDB(playlists=[])) + result = auto_discover_playlist({'all': True}, deps) + assert result == {'status': 'error', 'reason': 'No playlists found'} + + +# ─── refresh_mirrored ──────────────────────────────────────────────── + + +@dataclass +class _StubSpotifyTrack: + id: str + name: str + artists: list + album: str + duration_ms: int + image_url: Optional[str] = None + + +@dataclass +class _StubSpotifyPlaylist: + tracks: list + + +class _StubSpotifyClient: + def __init__(self, playlist): + self._playlist = playlist + self._authenticated = True + + def is_spotify_authenticated(self) -> bool: + return self._authenticated + + def get_playlist_by_id(self, _source_id): + return self._playlist + + +class TestRefreshMirrored: + def test_no_playlist_specified_returns_error(self): + deps = _build_deps() + result = auto_refresh_mirrored({}, deps) + assert result == {'status': 'error', 'reason': 'No playlist specified'} + + def test_filters_unrefreshable_sources(self): + # Sources 'file' and 'beatport' have no API to refresh from. + db = _StubDB(playlists=[ + {'id': 1, 'name': 'F', 'source': 'file', 'source_playlist_id': '1'}, + {'id': 2, 'name': 'B', 'source': 'beatport', 'source_playlist_id': '2'}, + ]) + deps = _build_deps(get_database=lambda: db) + result = auto_refresh_mirrored({'all': True}, deps) + assert result['status'] == 'completed' + assert result['refreshed'] == '0' + assert db.mirror_calls == [] # nothing got pushed to DB + + def test_spotify_refresh_writes_to_db(self): + track = _StubSpotifyTrack( + id='track123', name='Hello', artists=['Adele'], + album='25', duration_ms=295000, + ) + playlist = _StubSpotifyPlaylist(tracks=[track]) + spotify = _StubSpotifyClient(playlist) + db = _StubDB(playlists=[ + {'id': 5, 'name': 'My Spot', 'source': 'spotify', + 'source_playlist_id': 'spot-id', 'profile_id': 1}, + ]) + deps = _build_deps(get_database=lambda: db, spotify_client=spotify) + result = auto_refresh_mirrored({'playlist_id': '5'}, deps) + assert result['status'] == 'completed' + assert result['refreshed'] == '1' + assert len(db.mirror_calls) == 1 + call = db.mirror_calls[0] + assert call['source'] == 'spotify' + assert call['source_playlist_id'] == 'spot-id' + assert call['name'] == 'My Spot' + assert len(call['tracks']) == 1 + # Spotify-source tracks should be auto-marked discovered. + extra = json.loads(call['tracks'][0]['extra_data']) + assert extra['discovered'] is True + assert extra['provider'] == 'spotify' + assert extra['matched_data']['id'] == 'track123' + + def test_per_playlist_exception_collected_into_errors(self): + # Force an exception by making the DB blow up on mirror_playlist. + class _ExplodingDB(_StubDB): + def mirror_playlist(self, **kwargs): + raise RuntimeError('db disk full') + + track = _StubSpotifyTrack(id='t', name='t', artists=['a'], album='a', duration_ms=0) + spotify = _StubSpotifyClient(_StubSpotifyPlaylist(tracks=[track])) + db = _ExplodingDB(playlists=[ + {'id': 1, 'name': 'X', 'source': 'spotify', 'source_playlist_id': 'spot'}, + ]) + deps = _build_deps(get_database=lambda: db, spotify_client=spotify) + result = auto_refresh_mirrored({'all': True}, deps) + # Error captured, status still 'completed' (handler returns counts). + assert result['status'] == 'completed' + assert result['errors'] == '1' + assert result['refreshed'] == '0' + + +# ─── sync_playlist ─────────────────────────────────────────────────── + + +class TestSyncPlaylist: + def test_no_playlist_id_returns_error(self): + deps = _build_deps() + result = auto_sync_playlist({}, deps) + assert result == {'status': 'error', 'reason': 'No playlist specified'} + + def test_playlist_not_found_returns_error(self): + deps = _build_deps(get_database=lambda: _StubDB(playlists=[])) + result = auto_sync_playlist({'playlist_id': '99'}, deps) + assert result == {'status': 'error', 'reason': 'Playlist not found'} + + def test_no_tracks_returns_error(self): + db = _StubDB(playlists=[{'id': 1, 'name': 'P'}], playlist_tracks={1: []}) + deps = _build_deps(get_database=lambda: db) + result = auto_sync_playlist({'playlist_id': '1'}, deps) + assert result == {'status': 'error', 'reason': 'No tracks in playlist'} + + def test_no_discovered_tracks_skips(self): + # All tracks lack discovery + spotify_hint + valid IDs. + db = _StubDB( + playlists=[{'id': 1, 'name': 'P'}], + playlist_tracks={1: [{}, {}]}, # empty tracks → nothing usable + ) + deps = _build_deps(get_database=lambda: db) + result = auto_sync_playlist({'playlist_id': '1'}, deps) + assert result['status'] == 'skipped' + assert 'No discovered tracks' in result['reason'] + assert result['skipped_tracks'] == '2' + + def test_discovered_track_starts_sync_thread(self): + discovered_track = { + 'extra_data': json.dumps({ + 'discovered': True, + 'matched_data': { + 'id': 'spot-1', 'name': 'Track', 'artists': [{'name': 'X'}], + 'album': {'name': 'Album'}, 'duration_ms': 200000, + }, + }), + 'artist_name': 'X', + } + db = _StubDB( + playlists=[{'id': 1, 'name': 'P'}], + playlist_tracks={1: [discovered_track]}, + ) + sync_calls: List[tuple] = [] + deps = _build_deps( + get_database=lambda: db, + run_sync_task=lambda *a, **k: sync_calls.append((a, k)), + ) + result = auto_sync_playlist({'playlist_id': '1'}, deps) + assert result['status'] == 'started' + assert result['_manages_own_progress'] is True + assert result['discovered_tracks'] == '1' + # Wait for thread to fire run_sync_task + for _ in range(50): + if sync_calls: + break + import time + time.sleep(0.01) + assert len(sync_calls) == 1 + + def test_unchanged_since_last_sync_returns_skipped(self): + discovered_track = { + 'extra_data': json.dumps({ + 'discovered': True, + 'matched_data': { + 'id': 'spot-1', 'name': 'T', 'artists': [{'name': 'X'}], + 'album': {'name': 'A'}, 'duration_ms': 0, + }, + }), + 'artist_name': 'X', + } + db = _StubDB( + playlists=[{'id': 1, 'name': 'P'}], + playlist_tracks={1: [discovered_track]}, + ) + + # Pre-populate the sync-status file with the EXPECTED hash so the + # preflight short-circuit fires. + import hashlib + expected_hash = hashlib.md5('spot-1'.encode()).hexdigest() + sync_statuses = { + 'auto_mirror_1': {'tracks_hash': expected_hash, 'matched_tracks': 1} + } + + deps = _build_deps( + get_database=lambda: db, + load_sync_status_file=lambda: sync_statuses, + ) + result = auto_sync_playlist({'playlist_id': '1'}, deps) + assert result['status'] == 'skipped' + assert 'unchanged' in result['reason'] + + +# ─── playlist_pipeline ─────────────────────────────────────────────── + + +class TestPlaylistPipeline: + def test_no_playlist_specified_returns_error(self): + deps = _build_deps() + result = auto_playlist_pipeline({}, deps) + assert result == {'status': 'error', 'error': 'No playlist specified'} + # Pipeline-running flag MUST be cleared on error so the guard + # doesn't block subsequent triggers. + assert deps.state.pipeline_running is False + + def test_no_refreshable_playlists_clears_running_flag(self): + db = _StubDB(playlists=[ + {'id': 1, 'name': 'F', 'source': 'file'}, + {'id': 2, 'name': 'B', 'source': 'beatport'}, + ]) + deps = _build_deps(get_database=lambda: db) + result = auto_playlist_pipeline({'all': True}, deps) + assert result == {'status': 'error', 'error': 'No refreshable playlists found'} + assert deps.state.pipeline_running is False + + def test_pipeline_clears_running_on_unhandled_exception(self): + # Force the database accessor to blow up after the early checks. + class _ExplodingDB(_StubDB): + def get_mirrored_playlists(self): + raise RuntimeError('db down') + + db = _ExplodingDB(playlists=[]) + deps = _build_deps(get_database=lambda: db) + result = auto_playlist_pipeline({'all': True}, deps) + assert result['status'] == 'error' + assert result['_manages_own_progress'] is True + assert deps.state.pipeline_running is False diff --git a/tests/automation/test_handlers_simple.py b/tests/automation/test_handlers_simple.py new file mode 100644 index 00000000..ac8b2ef4 --- /dev/null +++ b/tests/automation/test_handlers_simple.py @@ -0,0 +1,329 @@ +"""Boundary tests for the simple extracted automation handlers +(``process_wishlist``, ``scan_watchlist``, ``scan_library``). + +Each handler is tested as a pure function: real ``AutomationDeps`` +constructed with stub callables, no Flask, no DB, no media-server +clients. The tests exercise the success path, the guard paths +(handler short-circuits when another instance is running), the +exception-swallowing contract (handlers must NEVER raise into the +engine), and the mutable-state machinery for handlers that own a +flag in ``AutomationState``. + +Pre-extraction these closures lived inside +``web_server._register_automation_handlers`` and were essentially +un-testable — every test would have needed to spin up the whole +Flask app and stub a dozen module-level globals.""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from typing import Any, Callable, List, Optional + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.process_wishlist import auto_process_wishlist +from core.automation.handlers.scan_watchlist import auto_scan_watchlist +from core.automation.handlers.scan_library import auto_scan_library + + +# ─── shared test scaffolding ────────────────────────────────────────── + + +def _build_deps(**overrides: Any) -> AutomationDeps: + """Return a default `AutomationDeps` with no-op callables. Tests + pass ``overrides`` to install behaviour on the specific deps they + care about.""" + + class _StubLogger: + def debug(self, *_a, **_k): pass + def info(self, *_a, **_k): pass + def warning(self, *_a, **_k): pass + def error(self, *_a, **_k): pass + + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=object(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── process_wishlist ───────────────────────────────────────────────── + + +class TestProcessWishlist: + def test_success_returns_completed_status(self): + called: List[Any] = [] + + def stub(automation_id=None): + called.append(automation_id) + + deps = _build_deps(process_wishlist_automatically=stub) + result = auto_process_wishlist({'_automation_id': 'auto-1'}, deps) + assert result == {'status': 'completed'} + assert called == ['auto-1'] + + def test_passes_none_when_no_automation_id(self): + called: List[Any] = [] + + def stub(automation_id=None): + called.append(automation_id) + + deps = _build_deps(process_wishlist_automatically=stub) + result = auto_process_wishlist({}, deps) + assert result == {'status': 'completed'} + assert called == [None] + + def test_handler_swallows_exceptions(self): + def stub(**_kwargs): + raise RuntimeError('boom') + + deps = _build_deps(process_wishlist_automatically=stub) + result = auto_process_wishlist({'_automation_id': 'a'}, deps) + assert result == {'status': 'error', 'error': 'boom'} + + +# ─── scan_watchlist ────────────────────────────────────────────────── + + +class TestScanWatchlist: + def test_fresh_scan_reports_summary_stats(self): + # Worker reassigns the state dict mid-run — handler detects + # via id() change and reports stats. + states = [ + {'summary': {}}, + {'summary': { + 'total_artists': 5, + 'successful_scans': 4, + 'new_tracks_found': 12, + 'tracks_added_to_wishlist': 8, + }}, + ] + idx = {'i': 0} + + def get_state(): + return states[idx['i']] + + def stub(**_kwargs): + idx['i'] = 1 # simulate the worker swapping the dict + + deps = _build_deps( + process_watchlist_scan_automatically=stub, + get_watchlist_scan_state=get_state, + ) + result = auto_scan_watchlist({}, deps) + assert result == { + 'status': 'completed', + 'artists_scanned': 5, + 'successful_scans': 4, + 'new_tracks_found': 12, + 'tracks_added_to_wishlist': 8, + } + + def test_no_fresh_scan_returns_bare_completed(self): + # Same dict identity before and after = no fresh scan ran. + same_dict = {'summary': {'total_artists': 999}} + deps = _build_deps( + process_watchlist_scan_automatically=lambda **_k: None, + get_watchlist_scan_state=lambda: same_dict, + ) + result = auto_scan_watchlist({}, deps) + assert result == {'status': 'completed'} + + def test_handler_swallows_exceptions(self): + def stub(**_kwargs): + raise ValueError('no scanner') + + deps = _build_deps(process_watchlist_scan_automatically=stub) + result = auto_scan_watchlist({}, deps) + assert result == {'status': 'error', 'error': 'no scanner'} + + +# ─── scan_library ──────────────────────────────────────────────────── + + +@dataclass +class _StubScanManager: + """Minimal fake of ``web_scan_manager`` — records calls + lets + tests script its responses.""" + + request_responses: List[dict] = field(default_factory=list) + status_responses: List[dict] = field(default_factory=list) + request_calls: List[str] = field(default_factory=list) + + def request_scan(self, label: str) -> dict: + self.request_calls.append(label) + return self.request_responses.pop(0) if self.request_responses else {'status': 'queued'} + + def get_scan_status(self) -> dict: + return self.status_responses.pop(0) if self.status_responses else {'status': 'idle'} + + +class TestScanLibrary: + def test_no_scan_manager_returns_error(self): + deps = _build_deps(web_scan_manager=None) + result = auto_scan_library({'_automation_id': 'a'}, deps) + assert result == {'status': 'error', 'reason': 'Scan manager not available'} + + def test_already_tracked_returns_skipped(self): + # Pre-set the state flag — handler should short-circuit. + state = AutomationState() + state.scan_library_automation_id = 'someone-else' + scanner = _StubScanManager(request_responses=[{'status': 'queued'}]) + deps = _build_deps(state=state, web_scan_manager=scanner) + result = auto_scan_library({'_automation_id': 'a'}, deps) + assert result == {'status': 'skipped', 'reason': 'Scan already being tracked'} + assert scanner.request_calls == ['Automation trigger (additional batch)'] + + def test_scan_completes_normally(self): + # request_scan returns scheduled; first poll = scheduled; + # second poll = scanning; third poll = idle. + scanner = _StubScanManager( + request_responses=[{'status': 'scheduled', 'delay_seconds': 5}], + status_responses=[ + {'status': 'scheduled'}, + {'status': 'scanning', 'elapsed_seconds': 10, 'max_time_seconds': 100}, + {'status': 'idle'}, + ], + ) + progress: List[dict] = [] + + def stub_progress(automation_id, **kwargs): + progress.append({'aid': automation_id, **kwargs}) + + deps = _build_deps( + web_scan_manager=scanner, + update_progress=stub_progress, + ) + # Patch time.sleep so the test runs instantly. + import core.automation.handlers.scan_library as module + original = module.time.sleep + module.time.sleep = lambda _: None + try: + result = auto_scan_library({'_automation_id': 'auto-1'}, deps) + finally: + module.time.sleep = original + + assert result['status'] == 'completed' + assert result.get('_manages_own_progress') is True + # State flag cleaned up after run + assert deps.state.scan_library_automation_id is None + # Progress phases emitted: scheduled, scan-start, scanning, completed + statuses = [p.get('status') for p in progress] + assert 'finished' in statuses + + def test_state_cleanup_on_exception(self): + class ExplodingScanner: + def request_scan(self, _): + raise RuntimeError('boom') + + def get_scan_status(self): + return {'status': 'idle'} + + progress: List[dict] = [] + deps = _build_deps( + web_scan_manager=ExplodingScanner(), + update_progress=lambda aid, **kw: progress.append({'aid': aid, **kw}), + ) + result = auto_scan_library({'_automation_id': 'auto-x'}, deps) + assert result['status'] == 'error' + assert result['_manages_own_progress'] is True + # State flag still cleaned up + assert deps.state.scan_library_automation_id is None + # Error progress emitted + assert any(p.get('status') == 'error' for p in progress) + + +# ─── AutomationState ────────────────────────────────────────────────── + + +class TestAutomationState: + def test_default_state(self): + s = AutomationState() + assert s.scan_library_automation_id is None + assert s.db_update_automation_id is None + assert s.pipeline_running is False + assert s.is_scan_library_active() is False + assert s.is_pipeline_running() is False + + def test_set_scan_library_id(self): + s = AutomationState() + s.set_scan_library_id('auto-1') + assert s.scan_library_automation_id == 'auto-1' + assert s.is_scan_library_active() is True + s.set_scan_library_id(None) + assert s.is_scan_library_active() is False + + def test_set_pipeline_running(self): + s = AutomationState() + s.set_pipeline_running(True) + assert s.is_pipeline_running() is True + s.set_pipeline_running(False) + assert s.is_pipeline_running() is False + + def test_concurrent_set_safe_via_lock(self): + # Smoke test: two threads flipping the same field don't crash. + # Lock ensures the final value is consistent. + s = AutomationState() + + def worker(): + for _ in range(100): + s.set_pipeline_running(True) + s.set_pipeline_running(False) + + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert s.pipeline_running is False diff --git a/tests/automation/test_progress_callbacks.py b/tests/automation/test_progress_callbacks.py new file mode 100644 index 00000000..10456bfd --- /dev/null +++ b/tests/automation/test_progress_callbacks.py @@ -0,0 +1,244 @@ +"""Boundary tests for the progress + history callbacks extracted +from ``web_server._register_automation_handlers``. + +The callbacks are wired by the engine via ``register_progress_callbacks``; +each test invokes the extracted top-level function with stub deps +and verifies the right downstream call fires.""" + +from __future__ import annotations + +import threading +from typing import Any, Dict, List, Tuple + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.progress_callbacks import ( + progress_init, + progress_finish, + record_history, + on_library_scan_completed, + register_library_scan_completed_emitter, +) + + +def _build_deps(**overrides) -> AutomationDeps: + class _StubLogger: + def debug(self, *a, **k): pass + def info(self, *a, **k): pass + def warning(self, *a, **k): pass + def error(self, *a, **k): pass + + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=object(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── progress_init ─────────────────────────────────────────────────── + + +class TestProgressInit: + def test_forwards_to_init_automation_progress(self): + captured: List[Tuple] = [] + + def fake(aid, name, action_type): + captured.append((aid, name, action_type)) + + deps = _build_deps(init_automation_progress=fake) + progress_init('auto-1', 'My Auto', 'wishlist', deps) + assert captured == [('auto-1', 'My Auto', 'wishlist')] + + +# ─── progress_finish ───────────────────────────────────────────────── + + +class TestProgressFinish: + def test_skips_when_handler_manages_own_progress(self): + # Handler set the flag — engine callback must NOT emit a + # second 'finished' over the top of the handler's own. + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda *a, **k: calls.append({'a': a, 'k': k})) + progress_finish('auto-1', {'_manages_own_progress': True, 'status': 'completed'}, deps) + assert calls == [] + + def test_completed_emits_finished_status(self): + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda aid, **kw: calls.append({'aid': aid, **kw})) + progress_finish('auto-1', {'status': 'completed'}, deps) + assert len(calls) == 1 + assert calls[0]['aid'] == 'auto-1' + assert calls[0]['status'] == 'finished' + assert calls[0]['progress'] == 100 + assert calls[0]['phase'] == 'Complete' + assert calls[0]['log_type'] == 'success' + + def test_error_status_emits_error_phase(self): + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda aid, **kw: calls.append({'aid': aid, **kw})) + progress_finish('auto-1', {'status': 'error', 'error': 'boom'}, deps) + assert calls[0]['status'] == 'error' + assert calls[0]['phase'] == 'Error' + assert calls[0]['log_line'] == 'boom' + assert calls[0]['log_type'] == 'error' + + def test_msg_falls_back_through_keys(self): + # error -> reason -> status -> 'done' + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda aid, **kw: calls.append({'aid': aid, **kw})) + progress_finish('auto-1', {'status': 'completed', 'reason': 'all good'}, deps) + assert calls[0]['log_line'] == 'all good' + + def test_msg_default_done(self): + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda aid, **kw: calls.append({'aid': aid, **kw})) + progress_finish('auto-1', {}, deps) + assert calls[0]['log_line'] == 'done' + + +# ─── record_history ────────────────────────────────────────────────── + + +class TestRecordHistory: + def test_passes_db_to_recorder(self): + captured: List[Tuple] = [] + db_obj = object() + deps = _build_deps( + get_database=lambda: db_obj, + record_progress_history=lambda aid, result, db: captured.append((aid, result, db)), + ) + record_history('auto-1', {'status': 'completed'}, deps) + assert captured == [('auto-1', {'status': 'completed'}, db_obj)] + + +# ─── on_library_scan_completed ─────────────────────────────────────── + + +class TestOnLibraryScanCompleted: + def test_no_engine_skips(self): + deps = _build_deps(engine=None) + # Should not raise. + on_library_scan_completed(deps) + + def test_emits_event_with_server_type(self): + emits: List[Tuple] = [] + + class _Engine: + def emit(self, name, payload): + emits.append((name, payload)) + + class _ScanMgr: + _current_server_type = 'plex' + + deps = _build_deps(engine=_Engine(), web_scan_manager=_ScanMgr()) + on_library_scan_completed(deps) + assert emits == [('library_scan_completed', {'server_type': 'plex'})] + + def test_unknown_server_type_when_attr_missing(self): + emits: List[Tuple] = [] + + class _Engine: + def emit(self, name, payload): + emits.append((name, payload)) + + deps = _build_deps(engine=_Engine(), web_scan_manager=object()) + on_library_scan_completed(deps) + assert emits[0][1] == {'server_type': 'unknown'} + + +# ─── register_library_scan_completed_emitter ───────────────────────── + + +class TestRegisterEmitter: + def test_no_scan_manager_noop(self): + # No web_scan_manager → no callback registered, no error. + deps = _build_deps(web_scan_manager=None) + register_library_scan_completed_emitter(deps) + + def test_registers_callback_with_scan_manager(self): + callbacks: List = [] + + class _ScanMgr: + _current_server_type = 'plex' + def add_scan_completion_callback(self, cb): + callbacks.append(cb) + + deps = _build_deps(web_scan_manager=_ScanMgr()) + register_library_scan_completed_emitter(deps) + assert len(callbacks) == 1 + # The registered callback must invoke without args (web_scan_manager + # calls completion callbacks with no params). + # Verify it does fire on_library_scan_completed when invoked. + emits: List = [] + + class _Engine: + def emit(self, name, payload): + emits.append((name, payload)) + + deps2 = _build_deps(engine=_Engine(), web_scan_manager=_ScanMgr()) + register_library_scan_completed_emitter(deps2) + # The lambda captured deps2; we need to grab the registered + # callback to invoke it. Re-register and capture. + captured = [] + class _Mgr2: + _current_server_type = 'jellyfin' + def add_scan_completion_callback(self, cb): + captured.append(cb) + deps3 = _build_deps(engine=_Engine(), web_scan_manager=_Mgr2()) + emits3 = [] + deps3 = _build_deps( + engine=type('E', (), {'emit': lambda self, n, p: emits3.append((n, p))})(), + web_scan_manager=_Mgr2(), + ) + register_library_scan_completed_emitter(deps3) + captured[0]() # invoke the registered callback + assert emits3 == [('library_scan_completed', {'server_type': 'jellyfin'})] diff --git a/tests/discovery/test_similar_artists_library_filter.py b/tests/discovery/test_similar_artists_library_filter.py new file mode 100644 index 00000000..3a379289 --- /dev/null +++ b/tests/discovery/test_similar_artists_library_filter.py @@ -0,0 +1,114 @@ +from database.music_database import MusicDatabase + + +def _names(artists): + return {artist.similar_artist_name for artist in artists} + + +def test_top_similar_artists_can_exclude_active_server_library_artists(tmp_path): + db = MusicDatabase(str(tmp_path / "music.db")) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="Owned By Spotify ID", + similar_artist_spotify_id="sp-owned", + profile_id=1, + ) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="Owned By Deezer ID", + similar_artist_deezer_id="dz-owned", + profile_id=1, + ) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="Owned By MusicBrainz ID", + similar_artist_musicbrainz_id="mb-owned", + profile_id=1, + ) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="Owned By Name", + similar_artist_spotify_id="sp-owned-name", + profile_id=1, + ) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="Different Server Artist", + similar_artist_spotify_id="sp-other-server", + profile_id=1, + ) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="Fresh Artist", + similar_artist_spotify_id="sp-fresh", + profile_id=1, + ) + + with db._get_connection() as conn: + conn.executemany( + """ + INSERT INTO artists (name, server_source, spotify_artist_id, deezer_id, musicbrainz_id) + VALUES (?, ?, ?, ?, ?) + """, + [ + ("Library Alias", "navidrome", "sp-owned", None, None), + ("Library Deezer Alias", "navidrome", None, "dz-owned", None), + ("Library MusicBrainz Alias", "navidrome", None, None, "mb-owned"), + ("owned by name", "navidrome", None, None, None), + ("Different Server Artist", "plex", "sp-other-server", None, None), + ], + ) + conn.commit() + + artists = db.get_top_similar_artists( + limit=20, + profile_id=1, + exclude_library_server="navidrome", + ) + + assert _names(artists) == {"Different Server Artist", "Fresh Artist"} + + +def test_top_similar_artists_can_require_musicbrainz_source(tmp_path): + db = MusicDatabase(str(tmp_path / "music.db")) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="MB Artist", + similar_artist_musicbrainz_id="mb-artist", + profile_id=1, + ) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="Spotify Only", + similar_artist_spotify_id="sp-artist", + profile_id=1, + ) + + artists = db.get_top_similar_artists(limit=20, profile_id=1, require_source="musicbrainz") + + assert _names(artists) == {"MB Artist"} + assert artists[0].similar_artist_musicbrainz_id == "mb-artist" + + +def test_top_similar_artists_keeps_existing_behavior_without_library_filter(tmp_path): + db = MusicDatabase(str(tmp_path / "music.db")) + db.add_or_update_similar_artist( + source_artist_id="seed-1", + similar_artist_name="Owned Artist", + similar_artist_spotify_id="sp-owned", + profile_id=1, + ) + + with db._get_connection() as conn: + conn.execute( + """ + INSERT INTO artists (name, server_source, spotify_artist_id) + VALUES (?, ?, ?) + """, + ("Owned Artist", "navidrome", "sp-owned"), + ) + conn.commit() + + artists = db.get_top_similar_artists(limit=20, profile_id=1) + + assert _names(artists) == {"Owned Artist"} diff --git a/tests/downloads/test_download_orchestrator.py b/tests/downloads/test_download_orchestrator.py index e157047e..b551750e 100644 --- a/tests/downloads/test_download_orchestrator.py +++ b/tests/downloads/test_download_orchestrator.py @@ -198,6 +198,37 @@ def test_reload_instances_with_no_args_reloads_every_source(): assert b.reload_called is True +def test_reload_settings_refreshes_registry_plugins(monkeypatch): + """Settings saves should refresh plugins that cache config at init. + + Prowlarr-backed torrent / usenet clients keep a ProwlarrClient + instance, so without this hook newly-saved indexer settings only + took effect after process restart. + """ + + class _ReloadSettingsClient(_FakeClient): + def __init__(self): + super().__init__() + self.reload_calls = 0 + + def reload_settings(self): + self.reload_calls += 1 + + torrent = _ReloadSettingsClient() + usenet = _ReloadSettingsClient() + orch = _build_orchestrator(torrent=torrent, usenet=usenet) + + monkeypatch.setattr( + 'core.download_orchestrator.config_manager.get', + lambda _key, default=None: default, + ) + + orch.reload_settings() + + assert torrent.reload_calls == 1 + assert usenet.reload_calls == 1 + + # --------------------------------------------------------------------------- # Singleton factory (matches Cin's get_metadata_engine pattern) # --------------------------------------------------------------------------- diff --git a/tests/downloads/test_downloads_candidates.py b/tests/downloads/test_downloads_candidates.py index 19471384..1de44b51 100644 --- a/tests/downloads/test_downloads_candidates.py +++ b/tests/downloads/test_downloads_candidates.py @@ -38,6 +38,10 @@ class _Candidate: title: str = "Song" artist: str = "Artist" album: str = "Album" + quality_score: float = 0.0 + upload_speed: int = 0 + queue_length: int = 0 + free_upload_slots: int = 0 @dataclass @@ -413,3 +417,20 @@ def test_candidates_with_equal_confidence_both_tried(): # First one wins — second never tried because download succeeded assert len(deps.download_orchestrator.download_calls) == 1 assert deps.download_orchestrator.download_calls[0][1] == "a.flac" + + +def test_equal_confidence_candidates_prefer_better_peer_quality(): + """Equal-confidence Soulseek candidates use peer quality as the tiebreaker.""" + deps = _build_deps() + _seed_task("t14") + candidates = [ + _Candidate(filename="slow.flac", confidence=0.9, quality_score=0.8, + upload_speed=100_000, queue_length=0, free_upload_slots=1), + _Candidate(filename="fast.flac", confidence=0.9, quality_score=1.0, + upload_speed=5_000_000, queue_length=0, free_upload_slots=1), + ] + track = _Track() + + dc.attempt_download_with_candidates("t14", candidates, track, batch_id=None, deps=deps) + + assert deps.download_orchestrator.download_calls[0][1] == "fast.flac" diff --git a/tests/downloads/test_downloads_lifecycle.py b/tests/downloads/test_downloads_lifecycle.py index fb67d157..8be74876 100644 --- a/tests/downloads/test_downloads_lifecycle.py +++ b/tests/downloads/test_downloads_lifecycle.py @@ -329,6 +329,48 @@ def test_batch_completion_emits_batch_complete_when_all_done(): assert 'b1' in monitor.stopped +def test_batch_completion_cleans_private_album_bundle_staging(tmp_path): + staging_dir = tmp_path / 'b1' + staging_dir.mkdir() + (staging_dir / 'leftover.flac').write_bytes(b'audio') + + download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}} + download_batches['b1'] = { + 'queue': ['t1'], 'queue_index': 1, 'active_count': 1, + 'max_concurrent': 1, 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'album_bundle_private_staging': True, + 'album_bundle_source': 'torrent', + 'album_bundle_staging_path': str(staging_dir), + } + deps, _ = _build_deps() + + lc.on_download_completed('b1', 't1', True, deps) + + assert not staging_dir.exists() + + +def test_batch_completion_keeps_unexpected_staging_path(tmp_path): + staging_dir = tmp_path / 'shared-staging' + staging_dir.mkdir() + (staging_dir / 'leftover.flac').write_bytes(b'audio') + + download_tasks['t1'] = {'status': 'completed', 'track_info': {'name': 'X'}} + download_batches['b1'] = { + 'queue': ['t1'], 'queue_index': 1, 'active_count': 1, + 'max_concurrent': 1, 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'album_bundle_private_staging': True, + 'album_bundle_source': 'torrent', + 'album_bundle_staging_path': str(staging_dir), + } + deps, _ = _build_deps() + + lc.on_download_completed('b1', 't1', True, deps) + + assert staging_dir.exists() + + def test_batch_completion_skips_emit_when_zero_successful(): """Don't emit batch_complete if nothing actually downloaded.""" download_tasks['t1'] = {'status': 'failed', 'track_info': {'name': 'X'}, 'track_index': 0} diff --git a/tests/downloads/test_downloads_master.py b/tests/downloads/test_downloads_master.py index e702e71e..a1586028 100644 --- a/tests/downloads/test_downloads_master.py +++ b/tests/downloads/test_downloads_master.py @@ -3,6 +3,7 @@ from __future__ import annotations import threading +from types import SimpleNamespace import pytest @@ -42,6 +43,7 @@ class _FakeDB: self.album_confidence = album_confidence self.sync_history_calls = [] self.track_results_calls = [] + self.manual_matches = [] def check_track_exists(self, title, artist, confidence_threshold=0.7, server_source=None, album=None): key = (title.lower().strip(), artist.lower().strip()) @@ -70,6 +72,23 @@ class _FakeDB: def update_sync_history_track_results(self, batch_id, results_json): self.track_results_calls.append((batch_id, results_json)) + def get_manual_library_match(self, profile_id, source, source_track_id, server_source=''): + for match in self.manual_matches: + if ( + match["profile_id"] == profile_id + and match["source"] == source + and match["source_track_id"] == source_track_id + and match.get("server_source", "") == server_source + ): + return match + return None + + def find_manual_library_match_by_source_track_id(self, profile_id, source_track_id, server_source=''): + for match in self.manual_matches: + if match["profile_id"] == profile_id and match["source_track_id"] == source_track_id: + return match + return None + class _DBTrack: def __init__(self, title): @@ -109,6 +128,40 @@ class _FakeSoulseekWrapper: def __init__(self, inner): self.soulseek = inner + def client(self, name): + return self.soulseek if name == 'soulseek' else None + + +class _FakePluginWrapper: + def __init__(self, plugins): + self._plugins = dict(plugins) + + def client(self, name): + return self._plugins.get(name) + + +class _FakeAlbumBundleSoulseek: + def __init__(self, outcome=None): + self.calls = [] + self.outcome = outcome or {'success': True, 'files': ['/tmp/a.flac']} + + def download_album_to_staging(self, album, artist, staging, emit, **kwargs): + self.calls.append((album, artist, staging, kwargs)) + emit({'state': 'staged', 'count': len(self.outcome.get('files', []))}) + return self.outcome + + +class _FakePreflightAlbumBundleSoulseek(_FakeSoulseek): + def __init__(self, *args, outcome=None, **kwargs): + super().__init__(*args, **kwargs) + self.calls = [] + self.outcome = outcome or {'success': True, 'files': ['/tmp/a.flac']} + + def download_album_to_staging(self, album, artist, staging, emit, **kwargs): + self.calls.append((album, artist, staging, kwargs)) + emit({'state': 'staged', 'count': len(self.outcome.get('files', []))}) + return self.outcome + class _FakeMonitor: def __init__(self): @@ -220,6 +273,38 @@ def _seed_batch(batch_id, **overrides): download_batches[batch_id] = base +def _slsk_track(title, number, folder='Artist/Test Album'): + return SimpleNamespace( + username='peer', + filename=f'{folder}/{number:02d} - {title}.flac', + title=title, + track_number=number, + quality='flac', + bitrate=None, + duration=180000, + size=20_000_000, + free_upload_slots=1, + upload_speed=2_000_000, + queue_length=0, + quality_score=1.0, + ) + + +def _album_result(username, path, title, tracks, *, artist='Artist', year='2020', + quality_score=0.9): + return SimpleNamespace( + username=username, + album_path=path, + album_title=title, + artist=artist, + year=year, + track_count=len(tracks), + tracks=tracks, + dominant_quality='flac', + quality_score=quality_score, + ) + + # --------------------------------------------------------------------------- # PHASE 1: analysis # --------------------------------------------------------------------------- @@ -260,6 +345,94 @@ def test_force_download_treats_all_as_missing(monkeypatch): assert download_batches['B2']['phase'] == 'downloading' +def test_manual_match_overrides_internal_force_download(monkeypatch): + """Internal wishlist force mode still honors explicit manual library matches.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + monkeypatch.setattr( + 'core.library.manual_library_match.get_match_for_track', + lambda *_args, **_kwargs: {'id': 1, 'library_track_id': 42}, + ) + + removed = [] + _seed_batch( + 'B2a', + force_download_all=True, + ignore_manual_matches=False, + profile_id=1, + batch_source='spotify', + ) + deps = _build_deps(wishlist_remove=lambda td: removed.append(td.get('name'))) + tracks = [{'id': 'spotify-track-1', 'name': 'T1', 'artists': ['A']}] + + mw.run_full_missing_tracks_process('B2a', 'wishlist', tracks, deps) + + assert download_batches['B2a']['queue'] == [] + assert download_batches['B2a']['analysis_results'][0]['found'] is True + assert download_batches['B2a']['analysis_results'][0]['match_reason'] == 'manual_library_match' + assert removed == ['T1'] + + +def test_manual_match_saved_under_mirrored_source_overrides_wishlist_batch(monkeypatch): + """Wishlist batches honor matches saved from mirrored sync history.""" + db = _FakeDB() + db.manual_matches.append({ + "id": 1, + "profile_id": 1, + "source": "mirrored", + "source_track_id": "track-abc", + "server_source": "", + "library_track_id": 42, + }) + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + removed = [] + _seed_batch( + 'B2m', + force_download_all=True, + ignore_manual_matches=False, + profile_id=1, + batch_source='wishlist', + ) + deps = _build_deps(wishlist_remove=lambda td: removed.append(td.get('name'))) + tracks = [{'id': 'track-abc', 'name': 'Coffee Break', 'artists': [{'name': 'Zeds Dead'}], 'provider': 'wishlist'}] + + mw.run_full_missing_tracks_process('B2m', 'wishlist', tracks, deps) + + assert download_batches['B2m']['queue'] == [] + assert download_batches['B2m']['analysis_results'][0]['found'] is True + assert download_batches['B2m']['analysis_results'][0]['match_reason'] == 'manual_library_match' + assert removed == ['Coffee Break'] + + +def test_explicit_force_download_ignores_manual_match(monkeypatch): + """User-facing Force Download All can intentionally bypass manual matches.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + calls = [] + monkeypatch.setattr( + 'core.library.manual_library_match.get_match_for_track', + lambda *_args, **_kwargs: calls.append(True) or {'id': 1, 'library_track_id': 42}, + ) + + _seed_batch( + 'B2b', + force_download_all=True, + ignore_manual_matches=True, + profile_id=1, + batch_source='spotify', + ) + deps = _build_deps() + tracks = [{'id': 'spotify-track-1', 'name': 'T1', 'artists': ['A']}] + + mw.run_full_missing_tracks_process('B2b', 'playlist1', tracks, deps) + + assert calls == [] + assert len(download_batches['B2b']['queue']) == 1 + assert download_batches['B2b']['analysis_results'][0]['found'] is False + + def test_found_tracks_trigger_wishlist_removal(monkeypatch): """When DB lookup succeeds, master worker invokes wishlist removal callback.""" db = _FakeDB(found_tracks={('t1', 'a'): 0.9}) @@ -454,6 +627,241 @@ def test_mb_release_preflight_skipped_when_no_mb_worker(monkeypatch): assert cache == {} # nothing cached +# --------------------------------------------------------------------------- +# Soulseek album preflight +# --------------------------------------------------------------------------- + +def test_soulseek_album_preflight_scores_release_folder_over_larger_wrong_edition(monkeypatch): + """Album preflight chooses the folder whose tracklist matches the target release.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + target_tracks = [ + {'name': f'Track {i}', 'artists': ['Artist'], 'track_number': i} + for i in range(1, 11) + ] + correct_tracks = [_slsk_track(f'Track {i}', i, folder='Artist/Test Album (2020)') for i in range(1, 11)] + wrong_tracks = [ + _slsk_track(f'Remix {i}', i, folder='Artist/Test Album Deluxe Remixes (2020)') + for i in range(1, 13) + ] + wrong = _album_result( + 'slow-peer', + 'Artist/Test Album Deluxe Remixes (2020)', + 'Test Album Deluxe Remixes', + wrong_tracks, + quality_score=1.0, + ) + correct = _album_result( + 'good-peer', + 'Artist/Test Album (2020)', + 'Test Album', + correct_tracks, + quality_score=0.8, + ) + slsk = _FakeSoulseek(album_results=[wrong, correct], browse_files=[{'filename': 'x.flac'}], + parsed_tracks=correct_tracks) + deps = _build_deps( + config=_FakeConfig({'download_source.mode': 'soulseek'}), + soulseek=_FakeSoulseekWrapper(slsk), + ) + _seed_batch( + 'B22', + is_album_download=True, + album_context={'name': 'Test Album', 'total_tracks': 10, 'release_date': '2020-01-01'}, + artist_context={'name': 'Artist'}, + ) + + mw.run_full_missing_tracks_process('B22', 'album:1', target_tracks, deps) + + assert download_batches['B22']['last_good_source'] == { + 'username': 'good-peer', + 'folder_path': 'Artist/Test Album (2020)', + } + assert download_batches['B22']['source_folder_tracks'] == correct_tracks + + +def test_soulseek_album_preflight_runs_when_soulseek_is_hybrid_primary(monkeypatch): + """Album preflight runs for hybrid album downloads when Soulseek is first.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + tracks = [{'name': 'T1', 'artists': ['Artist'], 'track_number': 1}] + folder_tracks = [_slsk_track('T1', 1)] + album = _album_result('peer', 'Artist/Test Album', 'Test Album', folder_tracks) + slsk = _FakeSoulseek(album_results=[album], browse_files=None, parsed_tracks=folder_tracks) + config = _FakeConfig({ + 'download_source.mode': 'hybrid', + 'download_source.hybrid_order': ['soulseek', 'hifi'], + }) + deps = _build_deps(config=config, soulseek=_FakeSoulseekWrapper(slsk)) + _seed_batch( + 'B23', + is_album_download=True, + album_context={'name': 'Test Album', 'total_tracks': 1}, + artist_context={'name': 'Artist'}, + ) + + mw.run_full_missing_tracks_process('B23', 'album:1', tracks, deps) + + assert slsk.search_calls + assert download_batches['B23']['last_good_source']['username'] == 'peer' + + +def test_soulseek_album_preflight_does_not_jump_ahead_of_hybrid_primary(monkeypatch): + """If Soulseek is only a fallback source, album preflight does not preempt source order.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + tracks = [{'name': 'T1', 'artists': ['Artist'], 'track_number': 1}] + folder_tracks = [_slsk_track('T1', 1)] + album = _album_result('peer', 'Artist/Test Album', 'Test Album', folder_tracks) + slsk = _FakeSoulseek(album_results=[album], browse_files=None, parsed_tracks=folder_tracks) + config = _FakeConfig({ + 'download_source.mode': 'hybrid', + 'download_source.hybrid_order': ['deezer_dl', 'soulseek'], + }) + deps = _build_deps(config=config, soulseek=_FakeSoulseekWrapper(slsk)) + _seed_batch( + 'B24', + is_album_download=True, + album_context={'name': 'Test Album', 'total_tracks': 1}, + artist_context={'name': 'Artist'}, + ) + + mw.run_full_missing_tracks_process('B24', 'album:1', tracks, deps) + + assert slsk.search_calls == [] + assert 'last_good_source' not in download_batches['B24'] + + +def test_soulseek_album_bundle_runs_after_missing_analysis(monkeypatch): + """Soulseek whole-folder bundles should engage only after analysis + has confirmed there is something missing.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + plugin = _FakeAlbumBundleSoulseek() + deps = _build_deps( + config=_FakeConfig({'download_source.mode': 'soulseek'}), + soulseek=_FakeSoulseekWrapper(plugin), + ) + _seed_batch( + 'B25', + is_album_download=True, + album_context={'name': 'Test Album', 'total_tracks': 1}, + artist_context={'name': 'Artist'}, + ) + tracks = [{'name': 'T1', 'artists': ['Artist'], 'track_number': 1}] + + mw.run_full_missing_tracks_process('B25', 'album:1', tracks, deps) + + assert len(plugin.calls) == 1 + album, artist, staging, kwargs = plugin.calls[0] + assert (album, artist) == ('Test Album', 'Artist') + assert staging.replace('\\', '/').endswith('storage/album_bundle_staging/B25') + assert kwargs == {} + assert download_batches['B25']['album_bundle_source'] == 'soulseek' + assert download_batches['B25']['album_bundle_private_staging'] is True + assert download_batches['B25']['album_bundle_state'] == 'staged' + assert 'last_good_source' not in download_batches['B25'] + + +def test_hybrid_first_soulseek_uses_album_bundle(monkeypatch): + """Hybrid keeps fallback semantics, but the first source can own + album-bundle downloads when it supports them.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + plugin = _FakeAlbumBundleSoulseek() + deps = _build_deps( + config=_FakeConfig({ + 'download_source.mode': 'hybrid', + 'download_source.hybrid_order': ['soulseek', 'hifi'], + }), + soulseek=_FakeSoulseekWrapper(plugin), + ) + _seed_batch( + 'B26', + is_album_download=True, + album_context={'name': 'Test Album', 'total_tracks': 1}, + artist_context={'name': 'Artist'}, + ) + tracks = [{'name': 'T1', 'artists': ['Artist'], 'track_number': 1}] + + mw.run_full_missing_tracks_process('B26', 'album:1', tracks, deps) + + assert len(plugin.calls) == 1 + assert download_batches['B26']['album_bundle_source'] == 'soulseek' + assert download_batches['B26']['album_bundle_private_staging'] is True + + +def test_soulseek_album_bundle_uses_preflight_source_without_preloading_reuse(monkeypatch): + """When the bundle path stages files, workers must claim staging + before any Soulseek source-reuse attempt can fire.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + tracks = [{'name': 'T1', 'artists': ['Artist'], 'track_number': 1}] + folder_tracks = [_slsk_track('T1', 1, folder='Artist/Test Album')] + album = _album_result('peer', 'Artist/Test Album', 'Test Album', folder_tracks) + slsk = _FakePreflightAlbumBundleSoulseek( + album_results=[album], + browse_files=None, + parsed_tracks=folder_tracks, + ) + deps = _build_deps( + config=_FakeConfig({'download_source.mode': 'soulseek'}), + soulseek=_FakeSoulseekWrapper(slsk), + ) + _seed_batch( + 'B28', + is_album_download=True, + album_context={'name': 'Test Album', 'total_tracks': 1}, + artist_context={'name': 'Artist'}, + ) + + mw.run_full_missing_tracks_process('B28', 'album:1', tracks, deps) + + assert len(slsk.calls) == 1 + assert slsk.calls[0][3] == { + 'preferred_source': { + 'username': 'peer', + 'folder_path': 'Artist/Test Album', + }, + 'preferred_tracks': folder_tracks, + } + assert download_batches['B28']['album_bundle_private_staging'] is True + assert 'last_good_source' not in download_batches['B28'] + assert 'source_folder_tracks' not in download_batches['B28'] + + +def test_hybrid_first_torrent_uses_album_bundle_before_per_track(monkeypatch): + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + plugin = _FakeAlbumBundleSoulseek() + deps = _build_deps( + config=_FakeConfig({ + 'download_source.mode': 'hybrid', + 'download_source.hybrid_order': ['torrent', 'soulseek'], + }), + soulseek=_FakePluginWrapper({'torrent': plugin}), + ) + _seed_batch( + 'B27', + is_album_download=True, + album_context={'name': 'Test Album', 'total_tracks': 1}, + artist_context={'name': 'Artist'}, + ) + tracks = [{'name': 'T1', 'artists': ['Artist'], 'track_number': 1}] + + mw.run_full_missing_tracks_process('B27', 'album:1', tracks, deps) + + assert len(plugin.calls) == 1 + assert download_batches['B27']['album_bundle_source'] == 'torrent' + + # --------------------------------------------------------------------------- # Task creation # --------------------------------------------------------------------------- @@ -538,6 +946,48 @@ def test_wishlist_album_grouping_resolves_artist(monkeypatch): assert 'Album Artist' in artist_names +def test_wishlist_album_grouping_uses_shared_rich_album_context(monkeypatch): + """Tracks in one wishlist album reuse the richest album context to avoid year folder splits.""" + db = _FakeDB() + monkeypatch.setattr('database.music_database.MusicDatabase', lambda: db) + + deps = _build_deps() + _seed_batch('B14b') + + tracks = [ + { + 'name': 'T1', 'artists': [{'name': 'Artist'}], + 'spotify_data': { + 'album': { + 'id': 'A1', 'name': 'Test Album', 'artists': [{'name': 'Album Artist'}], + 'release_date': '2024-05-05', 'total_tracks': 2, + 'album_type': 'album', 'images': [{'url': 'http://img'}], + }, + 'artists': [{'name': 'Artist'}], + }, + }, + { + 'name': 'T2', 'artists': [{'name': 'Artist'}], + 'spotify_data': { + 'album': {'id': 'A1', 'name': 'Test Album'}, + 'artists': [{'name': 'Artist'}], + }, + }, + ] + + mw.run_full_missing_tracks_process('B14b', 'wishlist', tracks, deps) + + release_dates = set() + images = set() + for tid in download_batches['B14b']['queue']: + album_ctx = download_tasks[tid]['track_info']['_explicit_album_context'] + release_dates.add(album_ctx['release_date']) + images.add(album_ctx['images'][0]['url']) + + assert release_dates == {'2024-05-05'} + assert images == {'http://img'} + + def test_playlist_folder_mode_propagates(monkeypatch): """Playlist folder mode flag carried through to track_info.""" db = _FakeDB() diff --git a/tests/downloads/test_downloads_post_processing.py b/tests/downloads/test_downloads_post_processing.py index a5f02d8c..455b2237 100644 --- a/tests/downloads/test_downloads_post_processing.py +++ b/tests/downloads/test_downloads_post_processing.py @@ -232,6 +232,28 @@ def test_file_found_in_downloads_with_context_runs_post_process_with_verificatio assert any(c[0] == 'post_process' for c in rec.calls) +def test_file_search_ignores_non_audio_candidates(monkeypatch): + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': 'Artist - Album.cue', + 'username': 'torrent', + 'track_info': {'name': 'Money'}, + } + matched_downloads_context['torrent::Artist - Album.cue'] = { + 'original_search_result': {'title': 'Money', 'track_number': 1}, + } + monkeypatch.setattr(pp.time, 'sleep', lambda s: None) + deps, rec = _build_deps( + find_completed_file=lambda *a, **kw: ('/downloads/Artist - Album.cue', 'download'), + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + assert download_tasks['t1']['status'] == 'failed' + assert not any(c[0] == 'post_process' for c in rec.calls) + assert ('on_complete', ('b1', 't1', False), {}) in rec.calls + + def test_file_found_in_downloads_no_context_marks_completed_directly(): """No matched context for the file → just mark completed since file exists.""" download_tasks['t1'] = { @@ -323,6 +345,139 @@ def test_youtube_task_uses_get_download_status_to_resolve_path(monkeypatch): assert any(c[0] == 'mark_completed' for c in rec.calls) +def test_torrent_release_copies_best_matching_audio_to_transfer(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '01 - Intro.flac' + right = release_dir / '02 - Money.flac' + wrong.write_bytes(b'wrong') + right.write_bytes(b'right') + transfer_dir = tmp_path / 'transfer' + + filename = 'magnet:?xt=abc||Artist - Album' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'Money', 'artists': [{'name': 'Artist'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Money', 'track_number': 2}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong), str(right)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + copied = transfer_dir / '02 - Money.flac' + assert copied.exists() + assert right.exists() + assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls) + + +def test_torrent_release_prefers_task_title_over_release_context(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '09.Harry Styles - Pop.flac' + right = release_dir / '10.Harry Styles - American Girls.flac' + wrong.write_bytes(b'wrong') + right.write_bytes(b'right') + transfer_dir = tmp_path / 'transfer' + + filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong), str(right)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + copied = transfer_dir / '10.Harry Styles - American Girls.flac' + assert copied.exists() + assert any(c[0] == 'post_process' and c[1][2] == str(copied) for c in rec.calls) + + +def test_torrent_release_without_matching_file_does_not_fallback_to_generic_search(tmp_path): + release_dir = tmp_path / 'release' + release_dir.mkdir() + wrong = release_dir / '09.Harry Styles - Pop.flac' + wrong.write_bytes(b'wrong') + transfer_dir = tmp_path / 'transfer' + + filename = 'http://prowlarr/download?id=1||Harry Styles - Kiss All The Time' + download_tasks['t1'] = { + 'status': 'post_processing', + 'filename': filename, + 'username': 'torrent', + 'download_id': 'dl-torrent-1', + 'track_info': {'name': 'American Girls', 'artists': [{'name': 'Harry Styles'}]}, + } + matched_downloads_context[f'torrent::{filename}'] = { + 'original_search_result': {'title': 'Pop', 'clean_title': 'Pop', 'track_number': 9}, + } + + class _FakeStatus: + file_path = str(wrong) + audio_files = [str(wrong)] + + class _FakeTorrentClient: + def get_download_status(self, dl_id): + assert dl_id == 'dl-torrent-1' + return _FakeStatus() + + def _unexpected_search(*args, **kwargs): + raise AssertionError("torrent releases should not fall back to generic file search") + + deps, rec = _build_deps( + config=_FakeConfig({'soulseek.transfer_path': str(transfer_dir)}), + download_orchestrator=_FakeTorrentClient(), + run_async=lambda coro: coro, + find_completed_file=_unexpected_search, + ) + + pp.run_post_processing_worker('t1', 'b1', deps) + + assert download_tasks['t1']['status'] == 'failed' + assert 'No matching audio file' in download_tasks['t1']['error_message'] + assert any(c[0] == 'on_complete' and c[1] == ('b1', 't1', False) for c in rec.calls) + assert not list(transfer_dir.glob('*')) + + def test_fuzzy_context_matching_when_exact_key_missing(monkeypatch): """When exact key isn't in matched_downloads_context, worker tries fuzzy match constrained to same Soulseek username.""" diff --git a/tests/downloads/test_downloads_staging.py b/tests/downloads/test_downloads_staging.py index d11153c4..70e86b6b 100644 --- a/tests/downloads/test_downloads_staging.py +++ b/tests/downloads/test_downloads_staging.py @@ -60,6 +60,7 @@ def _build_deps( transfer_path, staging_files=None, post_process_calls=None, + get_batch_field=None, ): post_process_calls = post_process_calls if post_process_calls is not None else [] deps = ds.StagingDeps( @@ -68,6 +69,7 @@ def _build_deps( get_staging_file_cache=lambda batch_id: staging_files or [], docker_resolve_path=lambda p: p, # passthrough post_process_matched_download_with_verification=lambda *a, **kw: post_process_calls.append((a, kw)), + get_batch_field=get_batch_field, ) deps._post_process_calls = post_process_calls return deps @@ -157,6 +159,56 @@ def test_exact_match_copies_to_transfer_and_marks_post_processing(tmp_path): assert context_key == 'staging_t4' +def test_private_album_bundle_staging_source_is_removed_after_claim(tmp_path): + src_file = tmp_path / 'private' / 'Hello.flac' + src_file.parent.mkdir() + src_file.write_bytes(b'fake audio') + + transfer_dir = tmp_path / 'transfer' + + def get_batch_field(_batch_id, field): + if field == 'album_bundle_source': + return 'torrent' + if field == 'album_bundle_private_staging': + return True + return None + + deps = _build_deps( + transfer_path=str(transfer_dir), + staging_files=[ + {'full_path': str(src_file), 'title': 'Hello', 'artist': 'Artist One'}, + ], + get_batch_field=get_batch_field, + ) + _seed_task('t_private') + + result = ds.try_staging_match('t_private', 'b_private', _Track(), deps) + + assert result is True + assert (transfer_dir / 'Hello.flac').exists() + assert not src_file.exists() + assert download_tasks['t_private']['username'] == 'torrent' + + +def test_public_staging_source_is_kept_after_match(tmp_path): + src_file = tmp_path / 'staging' / 'Hello.flac' + src_file.parent.mkdir() + src_file.write_bytes(b'fake audio') + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + {'full_path': str(src_file), 'title': 'Hello', 'artist': 'Artist One'}, + ], + ) + _seed_task('t_public') + + result = ds.try_staging_match('t_public', 'b_public', _Track(), deps) + + assert result is True + assert src_file.exists() + + def test_existing_file_in_transfer_gets_staging_suffix(tmp_path): """If destination already exists, suffix '_staging' added to avoid overwrite.""" src_file = tmp_path / 'staging' / 'Hello.flac' @@ -226,6 +278,180 @@ def test_explicit_album_context_uses_real_data(tmp_path): assert ctx['staging_source'] is True +def test_staging_context_falls_back_to_matched_file_track_number(tmp_path): + """Album-bundle staging can recover numbering from the selected audio file.""" + src_file = tmp_path / 'staging' / '03 - Backseat Freestyle.flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'Backseat Freestyle', + 'artist': 'Kendrick Lamar', + 'track_number': 3, + 'disc_number': 1, + }, + ], + ) + _seed_task('t6b', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + ds.try_staging_match( + 't6b', 'b1', + _Track(name='Backseat Freestyle', artists=['Kendrick Lamar']), + deps, + ) + + ctx = matched_downloads_context['staging_t6b'] + assert ctx['original_search_result']['track_number'] == 3 + assert ctx['original_search_result']['disc_number'] == 1 + + +def test_private_album_bundle_staging_overrides_default_track_info_number(tmp_path): + """Private release staging trusts the selected file number over weak task defaults.""" + src_file = tmp_path / 'staging' / '04-kendrick_lamar-the_art_of_peer_pressure.flac' + src_file.parent.mkdir() + src_file.touch() + + def get_batch_field(_batch_id, field): + if field == 'album_bundle_source': + return 'torrent' + if field == 'album_bundle_private_staging': + return True + return None + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'The Art of Peer Pressure', + 'artist': 'Kendrick Lamar', + }, + ], + get_batch_field=get_batch_field, + ) + _seed_task('t6c', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + 'track_number': 1, + }) + + ds.try_staging_match( + 't6c', 'b1', + _Track(name='The Art of Peer Pressure', artists=['Kendrick Lamar']), + deps, + ) + + ctx = matched_downloads_context['staging_t6c'] + assert ctx['track_info']['track_number'] == 4 + assert ctx['original_search_result']['track_number'] == 4 + assert ctx['original_search_result']['username'] == 'torrent' + assert ctx['original_search_result']['filename'] == str(src_file) + + +def test_staging_title_match_accepts_feature_suffix_from_release_file(tmp_path): + """Album releases can include featured artists in filenames.""" + src_file = tmp_path / 'staging' / '05-kendrick_lamar-money_trees_(feat._jay_rock).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'money_trees_(feat._jay_rock)', + 'artist': 'Kendrick Lamar', + 'track_number': 5, + }, + ], + ) + _seed_task('t_feature', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + result = ds.try_staging_match( + 't_feature', 'b1', + _Track(name='Money Trees', artists=['Kendrick Lamar']), + deps, + ) + + assert result is True + assert matched_downloads_context['staging_t_feature']['track_info']['track_number'] == 5 + + +def test_staging_title_match_accepts_bonus_track_against_release_file(tmp_path): + """Expected bonus labels should not block matching the actual release file.""" + src_file = tmp_path / 'staging' / '13-kendrick_lamar-the_recipe_(feat._dr._dre).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'the_recipe_(feat._dr._dre)', + 'artist': 'Kendrick Lamar', + 'track_number': 13, + }, + ], + ) + _seed_task('t_bonus', track_info={ + '_is_explicit_album_download': True, + '_explicit_album_context': {'id': 'alb', 'name': 'good kid, m.A.A.d city (Deluxe)'}, + '_explicit_artist_context': {'id': 'art', 'name': 'Kendrick Lamar'}, + }) + + result = ds.try_staging_match( + 't_bonus', 'b1', + _Track(name='The Recipe (Bonus Track)', artists=['Kendrick Lamar']), + deps, + ) + + assert result is True + assert matched_downloads_context['staging_t_bonus']['track_info']['track_number'] == 13 + + +def test_staging_title_match_keeps_wrong_versions_separate(tmp_path): + """Do not strip remix/extended wording when matching staged release files.""" + src_file = tmp_path / 'staging' / '17-kendrick_lamar-swimming_pools_(drank)_(black_hippy_remix).flac' + src_file.parent.mkdir() + src_file.touch() + + deps = _build_deps( + transfer_path=str(tmp_path / 'transfer'), + staging_files=[ + { + 'full_path': str(src_file), + 'title': 'swimming_pools_(drank)_(black_hippy_remix)', + 'artist': 'Kendrick Lamar', + 'track_number': 17, + }, + ], + ) + _seed_task('t_wrong_version') + + result = ds.try_staging_match( + 't_wrong_version', 'b1', + _Track(name='Swimming Pools (Drank) (Extended Version)', artists=['Kendrick Lamar']), + deps, + ) + + assert result is False + assert 'staging_t_wrong_version' not in matched_downloads_context + + def test_fallback_context_synthesizes_from_track(tmp_path): """Without explicit context, synthesizes spotify_artist/album from the track.""" src_file = tmp_path / 'staging' / 'Hello.flac' diff --git a/tests/downloads/test_downloads_status.py b/tests/downloads/test_downloads_status.py index 757f4ddc..706f9131 100644 --- a/tests/downloads/test_downloads_status.py +++ b/tests/downloads/test_downloads_status.py @@ -40,6 +40,8 @@ def _build_deps( make_key=None, submit_pp=None, cached_transfers=None, + download_orchestrator=None, + run_async=None, ): submitted = [] @@ -53,6 +55,8 @@ def _build_deps( make_context_key=make_key or (lambda u, f: f"{u}::{f}"), submit_post_processing=submit_pp or _default_submit, get_cached_transfer_data=cached_transfers or (lambda: {}), + download_orchestrator=download_orchestrator, + run_async=run_async, ) return deps, submitted @@ -85,6 +89,38 @@ def test_analysis_phase_includes_analysis_progress_and_results(): assert out['analysis_results'] == [{'track_index': 0, 'found': True}] +def test_album_downloading_phase_exposes_bundle_progress_without_task_safety_valve(): + deps, _ = _build_deps(config=_FakeConfig({'soulseek.download_timeout': 1})) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'queued', + 'track_info': {}, + 'status_change_time': 0, + } + batch = { + 'phase': 'album_downloading', + 'queue': ['t1'], + 'album_bundle_state': 'downloading', + 'album_bundle_source': 'torrent', + 'album_bundle_release': 'GNX [FLAC]', + 'album_bundle_progress': 0.42, + 'album_bundle_speed': 2048, + 'album_bundle_size': 4096, + } + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['album_bundle'] == { + 'state': 'downloading', + 'source': 'torrent', + 'release': 'GNX [FLAC]', + 'progress': 0.42, + 'progress_percent': 42, + 'speed': 2048, + 'size': 4096, + } + assert 'tasks' not in out + assert download_tasks['t1']['status'] == 'queued' + + def test_complete_phase_includes_wishlist_summary_when_present(): deps, _ = _build_deps() batch = { @@ -140,6 +176,7 @@ def test_task_status_includes_v2_state_fields(): 'cancel_requested': True, 'cancel_timestamp': 12345, 'ui_state': 'cancelling', 'playlist_id': 'pl1', 'error_message': 'oh no', 'cached_candidates': [{'x': 1}], + 'quarantine_entry_id': '20260514_120000_song', } batch = {'phase': 'downloading', 'queue': ['t1']} out = st.build_batch_status_data('b1', batch, {}, deps) @@ -149,6 +186,7 @@ def test_task_status_includes_v2_state_fields(): assert t['ui_state'] == 'cancelling' assert t['playlist_id'] == 'pl1' assert t['error_message'] == 'oh no' + assert t['quarantine_entry_id'] == '20260514_120000_song' assert t['has_candidates'] is True @@ -252,6 +290,128 @@ def test_post_processing_status_progress_is_95(): assert out['tasks'][0]['progress'] == 95 +def test_auto_torrent_without_live_entry_uses_engine_success_fallback(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + +def test_auto_torrent_prefers_engine_success_over_live_entry(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + live = {'torrent::song.flac': { + 'state': 'InProgress, Downloading', + 'percentComplete': 100, + }} + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, live, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + +def test_auto_torrent_engine_failure_does_not_bypass_monitor_retry(): + class _Record: + state = 'Completed, Errored' + progress = 100 + error = 'client failed' + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'downloading', + 'track_info': {}, + 'filename': 'song.flac', + 'username': 'torrent', + 'download_id': 'dl1', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'downloading' + assert download_tasks['t1']['status'] == 'downloading' + assert submitted == [] + + +def test_auto_torrent_engine_success_recovers_premature_failed_task(): + class _Record: + state = 'Completed, Succeeded' + progress = 100 + + class _Orchestrator: + def get_download_status(self, download_id): + assert download_id == 'dl1' + return _Record() + + deps, submitted = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda value: value, + ) + download_tasks['t1'] = { + 'track_index': 0, + 'status': 'failed', + 'track_info': {}, + 'filename': 'release-url||Release', + 'username': 'torrent', + 'download_id': 'dl1', + 'error_message': 'premature failure', + } + batch = {'phase': 'downloading', 'queue': ['t1']} + out = st.build_batch_status_data('b1', batch, {}, deps) + assert out['tasks'][0]['status'] == 'post_processing' + assert download_tasks['t1']['status'] == 'post_processing' + assert submitted == [('t1', 'b1')] + + # --------------------------------------------------------------------------- # Safety valve (stuck task handling) # --------------------------------------------------------------------------- @@ -338,6 +498,26 @@ def test_batched_status_no_filter_returns_all_batches(): assert set(out['batches'].keys()) == {'b1', 'b2'} +def test_unified_downloads_response_includes_album_bundle_summary(): + deps, _ = _build_deps() + download_batches['b1'] = { + 'phase': 'album_downloading', + 'playlist_id': 'pl1', + 'playlist_name': 'GNX', + 'queue': [], + 'album_bundle_state': 'downloading', + 'album_bundle_source': 'torrent', + 'album_bundle_progress': 75, + } + out = st.build_unified_downloads_response(300, deps) + assert out['batches'][0]['album_bundle'] == { + 'state': 'downloading', + 'source': 'torrent', + 'progress': 75, + 'progress_percent': 75, + } + + def test_batched_status_metadata_present(): deps, _ = _build_deps() download_batches['b1'] = {'phase': 'unknown'} diff --git a/tests/downloads/test_downloads_task_worker.py b/tests/downloads/test_downloads_task_worker.py index 11ccdfca..a222b311 100644 --- a/tests/downloads/test_downloads_task_worker.py +++ b/tests/downloads/test_downloads_task_worker.py @@ -5,14 +5,16 @@ from __future__ import annotations import pytest from core.downloads import task_worker as tw -from core.runtime_state import download_tasks +from core.runtime_state import download_batches, download_tasks @pytest.fixture(autouse=True) def reset_state(): download_tasks.clear() + download_batches.clear() yield download_tasks.clear() + download_batches.clear() # --------------------------------------------------------------------------- @@ -37,7 +39,7 @@ class _FakeClient: orchestrator); plain attrs (hybrid_order, hybrid_primary, etc.) are set as attributes so getattr() lookups still resolve them.""" _CLIENT_NAMES = {'soulseek', 'youtube', 'tidal', 'qobuz', 'hifi', - 'deezer_dl', 'lidarr', 'soundcloud'} + 'deezer_dl', 'lidarr', 'soundcloud', 'torrent', 'usenet'} def __init__(self, results=None, mode='soulseek', subclients=None): self._results = results if results is not None else [] @@ -54,7 +56,7 @@ class _FakeClient: def client(self, name): return self._client_map.get(name) - async def search(self, query, timeout=30): + async def search(self, query, timeout=30, exclude_sources=None): self.search_calls.append((query, timeout)) return (self._results, None) @@ -194,6 +196,119 @@ def test_staging_match_hit_returns_immediately(): assert rec.calls == [] +def test_private_torrent_album_staging_miss_skips_per_track_search(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Money Trees', 'artists': ['Kendrick Lamar'], + 'album': 'good kid, m.A.A.d city (Deluxe)', 'duration_ms': 387000, + }) + download_batches['b1'] = { + 'album_bundle_private_staging': True, + 'album_bundle_state': 'staged', + 'album_bundle_source': 'torrent', + } + client = _FakeClient(results=['should-not-search'], mode='torrent') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Kendrick Lamar Money Trees']), + try_staging_match=lambda *a, **kw: False, + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls == [] + assert download_tasks['t1']['status'] == 'not_found' + assert 'staged torrent album release' in download_tasks['t1']['error_message'] + assert ('done', ('b1', 't1', False), {}) in rec.calls + + +def test_private_soulseek_album_staging_miss_skips_per_track_search(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Song', 'artists': ['Artist'], + 'album': 'Album', 'duration_ms': 180000, + }) + download_batches['b1'] = { + 'album_bundle_private_staging': True, + 'album_bundle_state': 'staged', + 'album_bundle_source': 'soulseek', + } + client = _FakeClient(results=['should-not-search'], mode='soulseek') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Artist Song']), + try_staging_match=lambda *a, **kw: False, + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls == [] + assert download_tasks['t1']['status'] == 'not_found' + assert 'staged soulseek album release' in download_tasks['t1']['error_message'] + assert ('done', ('b1', 't1', False), {}) in rec.calls + + +def test_private_hybrid_first_soulseek_album_staging_miss_skips_per_track_search(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Song', 'artists': ['Artist'], + 'album': 'Album', 'duration_ms': 180000, + }) + download_batches['b1'] = { + 'album_bundle_private_staging': True, + 'album_bundle_state': 'staged', + 'album_bundle_source': 'soulseek', + } + client = _FakeClient( + results=['should-not-search'], + mode='hybrid', + subclients={'hybrid_order': ['soulseek', 'hifi']}, + ) + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Artist Song']), + try_staging_match=lambda *a, **kw: False, + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls == [] + assert download_tasks['t1']['status'] == 'not_found' + assert 'staged soulseek album release' in download_tasks['t1']['error_message'] + + +def test_partial_private_hybrid_first_soulseek_album_staging_miss_allows_per_track_search(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Song', 'artists': ['Artist'], + 'album': 'Album', 'duration_ms': 180000, + }) + download_batches['b1'] = { + 'album_bundle_private_staging': True, + 'album_bundle_state': 'staged', + 'album_bundle_source': 'soulseek', + 'album_bundle_partial': True, + } + client = _FakeClient( + results=[], + mode='hybrid', + subclients={'hybrid_order': ['soulseek', 'hifi']}, + ) + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Artist Song']), + try_staging_match=lambda *a, **kw: False, + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls + assert download_tasks['t1']['status'] == 'not_found' + assert 'staged soulseek album release' not in download_tasks['t1']['error_message'] + + # --------------------------------------------------------------------------- # Search loop happy path # --------------------------------------------------------------------------- @@ -219,6 +334,25 @@ def test_first_query_success_returns_after_storing_source(): assert download_tasks['t1']['status'] == 'searching' +def test_torrent_mode_uses_album_release_after_track_queries(): + _seed_task(track_info={ + 'id': 'sp-1', 'name': 'Money', 'artists': ['Pink Floyd'], + 'album': 'The Dark Side of the Moon', 'duration_ms': 383000, + }) + client = _FakeClient(results=[], mode='torrent') + rec = _Recorder() + deps, _ = _build_deps( + soulseek=client, + matching=_FakeMatchEngine(queries=['Pink Floyd Money']), + on_download_completed=rec('done'), + ) + + tw.download_track_worker('t1', 'b1', deps) + + assert client.search_calls[0][0] == 'Pink Floyd Money' + assert client.search_calls[-1][0] == 'Pink Floyd The Dark Side of the Moon' + + def test_no_results_marks_not_found_and_calls_completion(): _seed_task() rec = _Recorder() @@ -271,7 +405,7 @@ def test_cancellation_mid_query_returns_without_completion(): _seed_task() rec = _Recorder() - def _cancel_during_search(query, timeout=30): + def _cancel_during_search(query, timeout=30, exclude_sources=None): download_tasks['t1']['status'] = 'cancelled' async def _empty(): diff --git a/tests/downloads/test_downloads_validation.py b/tests/downloads/test_downloads_validation.py index 62798523..1acdf91c 100644 --- a/tests/downloads/test_downloads_validation.py +++ b/tests/downloads/test_downloads_validation.py @@ -11,19 +11,32 @@ from __future__ import annotations from dataclasses import dataclass from typing import Optional -from core.downloads.validation import filter_soundcloud_previews +from core.downloads import validation +from core.downloads.validation import filter_soundcloud_previews, get_valid_candidates @dataclass class _Track: duration_ms: int + name: str = 'Song' + artists: tuple[str, ...] = ('Artist',) @dataclass class _Candidate: username: str duration: Optional[int] # milliseconds - title: str = '' + title: str = 'Song' + artist: str = 'Artist' + filename: str = 'candidate' + + +class _MatchingEngine: + def score_track_match(self, **kwargs): + return 0.99, 'core_title_match' + + def normalize_string(self, text): + return (text or '').lower() def test_drops_soundcloud_30s_preview_when_expected_long(): @@ -90,3 +103,67 @@ def test_keeps_soundcloud_candidate_at_threshold(): # 110s passes both checks: > 35s AND > 100s (half of 200s) cand = _Candidate(username='soundcloud', duration=110_000) assert filter_soundcloud_previews([cand], expected) == [cand] + + +def test_rejects_tidal_candidate_that_would_fail_integrity_duration(monkeypatch): + """Structured sources should not download candidates that post-processing + will immediately quarantine for the same duration mismatch.""" + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=338_000) + wrong_tidal = _Candidate(username='tidal', duration=30_000) + + assert get_valid_candidates([wrong_tidal], expected, 'Artist Song') == [] + + +def test_keeps_tidal_candidate_inside_integrity_duration_tolerance(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=338_000) + tidal = _Candidate(username='tidal', duration=340_000) + + result = get_valid_candidates([tidal], expected, 'Artist Song') + + assert result == [tidal] + + +def test_rejects_torrent_title_match_from_wrong_artist(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + wrong_artist = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Tinkabelle', + ) + + assert get_valid_candidates([wrong_artist], expected, 'Olivia Dean The Man I Need') == [] + + +def test_keeps_torrent_title_match_from_expected_artist(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + correct_artist = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Olivia Dean', + ) + + result = get_valid_candidates([correct_artist], expected, 'Olivia Dean The Man I Need') + + assert result == [correct_artist] + + +def test_keeps_torrent_title_match_when_artist_is_indexer_fallback(monkeypatch): + monkeypatch.setattr(validation, 'matching_engine', _MatchingEngine()) + expected = _Track(duration_ms=180_000, name='The Man I Need', artists=('Olivia Dean',)) + candidate = _Candidate( + username='torrent', + duration=None, + title='The Man I Need', + artist='Indexer', + ) + candidate._source_metadata = {'indexer': 'Indexer'} + + result = get_valid_candidates([candidate], expected, 'Olivia Dean The Man I Need') + + assert result == [candidate] diff --git a/tests/downloads/test_hifi_pinning.py b/tests/downloads/test_hifi_pinning.py index 24ca15f1..d5fc9a8b 100644 --- a/tests/downloads/test_hifi_pinning.py +++ b/tests/downloads/test_hifi_pinning.py @@ -102,3 +102,153 @@ def test_cancel_download_marks_cancelled(hifi_client_with_engine): ok = _run_async(client.cancel_download('dl-1', None, remove=False)) assert ok is True assert engine.get_record('hifi', 'dl-1')['state'] == 'Cancelled' + + +def test_instance_capability_probe_uses_track_manifests_not_legacy_track(): + class _Response: + def __init__(self, *, ok=True, status_code=200, payload=None): + self.ok = ok + self.status_code = status_code + self._payload = payload or {} + + def json(self): + return self._payload + + class _Session: + def __init__(self): + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + if url.endswith('/search/'): + return _Response(payload={'items': []}) + if url.endswith('/trackManifests/'): + return _Response(payload={ + 'data': { + 'data': { + 'attributes': { + 'uri': 'https://cdn.example/playlist.m3u8', + }, + }, + }, + }) + if url.endswith('/'): + return _Response(payload={'version': 'test'}) + return _Response(ok=False, status_code=404) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + called_urls = [url for url, _ in client.session.calls] + assert result['can_search'] is True + assert result['can_download'] is True + assert any(url.endswith('/trackManifests/') for url in called_urls) + assert not any(url.endswith('/track') for url in called_urls) + + +def test_instance_capability_probe_reports_manifest_without_uri_as_limited(): + class _Response: + ok = True + status_code = 200 + + def __init__(self, payload): + self._payload = payload + + def json(self): + return self._payload + + class _Session: + def get(self, url, **kwargs): + if url.endswith('/search/'): + return _Response({'items': []}) + if url.endswith('/trackManifests/'): + return _Response({'data': {'data': {'attributes': {}}}}) + if url.endswith('/'): + return _Response({'version': 'test'}) + return _Response({'data': {'data': {'attributes': {}}}}) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + assert result['can_search'] is True + assert result['can_download'] is False + assert result['download_error'] == 'No playable manifest URL' + + +def test_instance_capability_probe_accepts_legacy_track_manifest(): + import base64 + import json + + class _Response: + def __init__(self, payload, *, ok=True, status_code=200): + self._payload = payload + self.ok = ok + self.status_code = status_code + + def json(self): + return self._payload + + class _Session: + def __init__(self): + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + if url.endswith('/search/'): + return _Response({'items': []}) + if url.endswith('/trackManifests/'): + return _Response({'data': {'data': {'attributes': {}}}}) + if url.endswith('/track/'): + manifest = base64.b64encode(json.dumps({ + 'mimeType': 'audio/flac', + 'codecs': 'flac', + 'encryptionType': 'NONE', + 'urls': ['https://cdn.example/track.flac'], + }).encode()).decode() + return _Response({'data': {'manifest': manifest}}) + if url.endswith('/'): + return _Response({'version': 'test'}) + return _Response({}, ok=False, status_code=404) + + client = HiFiClient.__new__(HiFiClient) + client.session = _Session() + + result = client.check_instance_capabilities('https://hifi.example') + + assert result['can_search'] is True + assert result['can_download'] is True + assert result['download_probe'] == 'track' + assert any(url.endswith('/track/') for url, _ in client.session.calls) + + +def test_get_hls_manifest_falls_back_to_legacy_track_endpoint(): + import base64 + import json + + client = HiFiClient.__new__(HiFiClient) + calls = [] + + def _fake_api_get(path, params=None, timeout=15): + calls.append((path, params)) + if path == '/trackManifests/': + return None + manifest = base64.b64encode(json.dumps({ + 'mimeType': 'audio/flac', + 'codecs': 'flac', + 'encryptionType': 'NONE', + 'urls': ['https://cdn.example/track.flac'], + }).encode()).decode() + return {'data': {'manifest': manifest}} + + client._api_get = _fake_api_get + + result = client._get_hls_manifest(123, quality='lossless') + + assert result['direct_urls'] == ['https://cdn.example/track.flac'] + assert result['extension'] == 'flac' + assert calls[0][0] == '/trackManifests/' + assert calls[1][0] == '/track/' diff --git a/tests/downloads/test_soulseek_pinning.py b/tests/downloads/test_soulseek_pinning.py index 253be224..3ba3508b 100644 --- a/tests/downloads/test_soulseek_pinning.py +++ b/tests/downloads/test_soulseek_pinning.py @@ -28,6 +28,7 @@ from unittest.mock import AsyncMock, patch import pytest from core.soulseek_client import SoulseekClient +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult def _run_async(coro): @@ -121,6 +122,153 @@ def test_download_extracts_id_from_dict_response(configured_client): assert result == 'abc123' +# --------------------------------------------------------------------------- +# album bundle +# --------------------------------------------------------------------------- + + +def _track(username='peer', filename='Artist/Album/01 - Song.flac', title='Song', number=1, size=10): + return TrackResult( + username=username, + filename=filename, + size=size, + bitrate=None, + duration=180000, + quality='flac', + free_upload_slots=1, + upload_speed=1_000_000, + queue_length=0, + artist='Artist', + title=title, + album='Album', + track_number=number, + ) + + +def test_album_bundle_stages_one_selected_soulseek_folder(configured_client, tmp_path): + configured_client.download_path = tmp_path + local_file = tmp_path / '01 - Song.flac' + local_file.write_bytes(b'audio') + track = _track(filename='Artist/Album/01 - Song.flac') + album = AlbumResult( + username='peer', + album_path='Artist/Album', + album_title='Album', + artist='Artist', + track_count=1, + total_size=10, + tracks=[track], + dominant_quality='flac', + free_upload_slots=1, + upload_speed=1_000_000, + queue_length=0, + ) + events = [] + + with patch.object(configured_client, 'search', AsyncMock(return_value=([], [album]))), \ + patch.object(configured_client, 'browse_user_directory', AsyncMock(return_value=[ + {'filename': '01 - Song.flac', 'size': 10} + ])), \ + patch.object(configured_client, 'filter_results_by_quality_preference', side_effect=lambda tracks: tracks), \ + patch.object(configured_client, 'download', AsyncMock(return_value='dl-1')) as download_mock, \ + patch.object(configured_client, 'get_all_downloads', AsyncMock(return_value=[ + DownloadStatus( + id='dl-1', + username='peer', + filename='Artist/Album/01 - Song.flac', + state='Completed, Succeeded', + progress=100, + size=10, + transferred=10, + speed=0, + ) + ])), \ + patch('core.soulseek_client.get_poll_timeout', return_value=1), \ + patch('core.soulseek_client.get_poll_interval', return_value=0.01): + outcome = configured_client.download_album_to_staging( + 'Album', + 'Artist', + str(tmp_path / 'staging'), + events.append, + ) + + assert outcome['success'] is True + assert outcome['fallback'] is False + assert len(outcome['files']) == 1 + assert Path(outcome['files'][0]).read_bytes() == b'audio' + download_mock.assert_awaited_once_with( + 'peer', + 'Artist/Album/01 - Song.flac', + 10, + ) + assert events[-1]['state'] == 'staged' + + +def test_album_bundle_stages_completed_files_when_same_source_partially_times_out(configured_client, tmp_path): + configured_client.download_path = tmp_path + (tmp_path / '01 - Ready.flac').write_bytes(b'audio') + ready = _track(filename='Artist/Album/01 - Ready.flac', title='Ready', number=1) + timed_out = _track(filename='Artist/Album/02 - Waiting.flac', title='Waiting', number=2) + events = [] + + with patch.object(configured_client, 'download', AsyncMock(side_effect=['dl-1', 'dl-2'])), \ + patch.object(configured_client, 'filter_results_by_quality_preference', side_effect=lambda tracks: tracks), \ + patch.object(configured_client, 'get_all_downloads', AsyncMock(return_value=[ + DownloadStatus( + id='dl-1', + username='peer', + filename='Artist/Album/01 - Ready.flac', + state='Completed, Succeeded', + progress=100, + size=10, + transferred=10, + speed=0, + ), + DownloadStatus( + id='dl-2', + username='peer', + filename='Artist/Album/02 - Waiting.flac', + state='TimedOut', + progress=0, + size=10, + transferred=0, + speed=0, + ), + ])), \ + patch('core.soulseek_client.get_poll_timeout', return_value=1), \ + patch('core.soulseek_client.get_poll_interval', return_value=0.01): + outcome = configured_client.download_album_to_staging( + 'Album', + 'Artist', + str(tmp_path / 'staging'), + events.append, + preferred_source={'username': 'peer', 'folder_path': 'Artist/Album'}, + preferred_tracks=[ready, timed_out], + ) + + assert outcome['success'] is True + assert outcome['fallback'] is False + assert len(outcome['files']) == 1 + assert Path(outcome['files'][0]).name == '01 - Ready.flac' + assert any(event.get('failed') == 1 for event in events) + + +def test_album_bundle_falls_back_when_no_album_folder(configured_client, tmp_path): + configured_client.download_path = tmp_path + with patch.object(configured_client, 'search', AsyncMock(return_value=([], []))), \ + patch.object(configured_client, 'download', AsyncMock(return_value='dl-1')) as dl: + outcome = configured_client.download_album_to_staging( + 'Missing Album', + 'Artist', + str(tmp_path / 'staging'), + ) + + assert outcome['success'] is False + assert outcome['fallback'] is True + assert 'No complete Soulseek album folders' in outcome['error'] + dl.assert_not_awaited() + + def test_download_extracts_id_from_list_response(configured_client): """Pinning: slskd sometimes returns a list of file objects. The first item's id is the download_id.""" @@ -347,3 +495,336 @@ def test_make_direct_request_returns_none_on_timeout(configured_client): result = _run_async(configured_client._make_direct_request('GET', 'health')) assert result is None + + +# --------------------------------------------------------------------------- +# Issue #649 — connection-error log spam suppression +# --------------------------------------------------------------------------- + + +def _build_unreachable_session(error_message: str = 'Cannot connect to host'): + """Stub aiohttp session whose request() raises ClientConnectorError.""" + import aiohttp + from unittest.mock import MagicMock + + class _Cm: + async def __aenter__(self_inner): + # ClientConnectorError needs a connection_key + OSError. The + # exact values don't matter for the test — we just need an + # instance of the right class so the except-branch fires. + os_err = OSError(-2, 'Name or service not known') + raise aiohttp.ClientConnectorError(MagicMock(), os_err) + async def __aexit__(self_inner, *args): + return None + + class _StubSession: + async def __aenter__(self): + return self + async def __aexit__(self, *args): + return None + def request(self, *args, **kwargs): + return _Cm() + async def close(self): + return None + + return _StubSession + + +def test_unreachable_slskd_returns_none_not_raises(configured_client): + """Pin: ClientConnectorError must not propagate. Caller treats None + as a normal failure (same as a 5xx) — every consumer that gates on + `if response is None` keeps working when slskd is unreachable.""" + StubSession = _build_unreachable_session() + with patch('aiohttp.ClientSession', return_value=StubSession()): + result = _run_async(configured_client._make_request('GET', 'transfers/downloads')) + assert result is None + + +def test_unreachable_slskd_logs_warning_once_then_debug(configured_client, caplog): + """Issue #649: status polling at /api/downloads/status fans out to + every plugin including soulseek even when the user has soulseek + toggled out, so each frontend poll produced an ERROR log line. Pin + that the FIRST unreachable response emits one WARNING with + actionable context, and subsequent repeats demote to DEBUG so the + log isn't spammed for the lifetime of every non-soulseek download.""" + import logging + configured_client._last_unreachable_logged = False + StubSession = _build_unreachable_session() + + with patch('aiohttp.ClientSession', return_value=StubSession()): + with caplog.at_level(logging.DEBUG, logger='soulseek_client'): + # Three repeated polls — first must warn, rest must stay quiet. + _run_async(configured_client._make_request('GET', 'transfers/downloads')) + _run_async(configured_client._make_request('GET', 'transfers/downloads')) + _run_async(configured_client._make_request('GET', 'transfers/downloads')) + + warning_records = [r for r in caplog.records if r.levelno == logging.WARNING + and 'slskd unreachable' in r.message] + error_records = [r for r in caplog.records if r.levelno == logging.ERROR + and 'Error making API request' in r.message] + assert len(warning_records) == 1, \ + f"Expected exactly 1 WARNING (one-time slskd-unreachable notice), got {len(warning_records)}" + assert len(error_records) == 0, \ + "Connection errors must not log at ERROR — that's the spam pattern #649 reported" + assert configured_client._last_unreachable_logged is True + + +def test_unreachable_flag_resets_on_successful_response(configured_client, caplog): + """When slskd comes back up after a stretch of being down, a fresh + WARNING should fire if it goes down again later — the suppression is + per-outage, not per-process-lifetime. The flag resets on any + successful (200/201/204) response.""" + import logging + configured_client._last_unreachable_logged = True # Simulate prior outage already warned + + # Simulate a 200 response — must reset the suppression flag. + class _OkCm: + async def __aenter__(self_inner): + class _Resp: + status = 200 + reason = 'OK' + async def text(self_resp): + return '{"ok": true}' + async def json(self_resp): + return {'ok': True} + return _Resp() + async def __aexit__(self_inner, *args): + return None + + class _OkSession: + async def __aenter__(self): + return self + async def __aexit__(self, *args): + return None + def request(self, *args, **kwargs): + return _OkCm() + async def close(self): + return None + + with patch('aiohttp.ClientSession', return_value=_OkSession()): + _run_async(configured_client._make_request('GET', 'server/state')) + + assert configured_client._last_unreachable_logged is False, \ + "Successful response must reset the suppression flag so a future outage warns again" + + +def test_make_direct_request_also_suppresses_unreachable_spam(configured_client, caplog): + """`_make_direct_request` shares the same base_url and same outage + mode, so it gets the same WARNING-once + DEBUG-after treatment.""" + import logging + configured_client._last_unreachable_logged = False + StubSession = _build_unreachable_session() + + with patch('aiohttp.ClientSession', return_value=StubSession()): + with caplog.at_level(logging.DEBUG, logger='soulseek_client'): + _run_async(configured_client._make_direct_request('GET', 'health')) + _run_async(configured_client._make_direct_request('GET', 'health')) + + warning_records = [r for r in caplog.records if r.levelno == logging.WARNING + and 'slskd unreachable' in r.message] + error_records = [r for r in caplog.records if r.levelno == logging.ERROR + and 'Error making direct API request' in r.message] + assert len(warning_records) == 1 + assert len(error_records) == 0 + + +def test_non_connection_exception_still_logs_error(configured_client, caplog): + """Guard: only ClientConnectorError gets the suppression treatment. + Any other exception (programming bug, unexpected aiohttp behaviour, + etc.) must still surface at ERROR so we don't accidentally hide + real problems behind the noise reduction.""" + import logging + + class _BoomCm: + async def __aenter__(self_inner): + raise ValueError("not a connection error — should still log ERROR") + async def __aexit__(self_inner, *args): + return None + + class _BoomSession: + async def __aenter__(self): + return self + async def __aexit__(self, *args): + return None + def request(self, *args, **kwargs): + return _BoomCm() + async def close(self): + return None + + with patch('aiohttp.ClientSession', return_value=_BoomSession()): + with caplog.at_level(logging.DEBUG, logger='soulseek_client'): + result = _run_async(configured_client._make_request('GET', 'transfers/downloads')) + + assert result is None # Still returns None — non-raising contract preserved + error_records = [r for r in caplog.records if r.levelno == logging.ERROR + and 'Error making API request' in r.message] + assert len(error_records) == 1, "Non-connection exceptions must still log ERROR" + + +# --------------------------------------------------------------------------- +# Issue #652 — quarantined-source dedup in the candidate filter +# --------------------------------------------------------------------------- + + +def _mk_track_result(username='peer', filename='song.flac', quality='flac', + bitrate=1411, size=10_000_000, duration=180_000): + """Build a minimal TrackResult for the candidate filter tests.""" + from core.download_plugins.types import TrackResult + return TrackResult( + username=username, + filename=filename, + size=size, + bitrate=bitrate, + duration=duration, + quality=quality, + free_upload_slots=1, + upload_speed=1_000_000, + queue_length=0, + ) + + +def test_drop_quarantined_sources_keeps_clean_candidates(configured_client, tmp_path, monkeypatch): + """When no candidate matches a quarantined `(username, filename)`, + every result passes through. Filter is a no-op for clean searches.""" + quarantine_dir = tmp_path / 'ss_quarantine' + quarantine_dir.mkdir() + + # Patch config_manager to point at our temp download path. + import core.soulseek_client as sc + monkeypatch.setattr(sc.config_manager, 'get', + lambda key, default=None: str(tmp_path) if key == 'soulseek.download_path' else default) + + results = [ + _mk_track_result(username='goodpeer1', filename='a.flac'), + _mk_track_result(username='goodpeer2', filename='b.flac'), + ] + + kept = configured_client._drop_quarantined_sources(results) + + assert len(kept) == 2 + assert {r.username for r in kept} == {'goodpeer1', 'goodpeer2'} + + +def test_drop_quarantined_sources_drops_known_bad(configured_client, tmp_path, monkeypatch): + """Issue #652 core contract: a candidate whose `(username, filename)` + matches a quarantined entry is dropped before the quality picker + ranks it. Stops the loop where the same source kept winning the + quality picker and re-downloading itself.""" + import json as _json + quarantine_dir = tmp_path / 'ss_quarantine' + quarantine_dir.mkdir() + + # Write a sidecar matching the bad source. + sidecar = { + "original_filename": "bad.flac", + "quarantine_reason": "AcoustID mismatch", + "context": { + "original_search_result": { + "username": "badpeer", "filename": "albums/bad.flac", + }, + }, + } + (quarantine_dir / "20260518_120000.json").write_text(_json.dumps(sidecar)) + + import core.soulseek_client as sc + monkeypatch.setattr(sc.config_manager, 'get', + lambda key, default=None: str(tmp_path) if key == 'soulseek.download_path' else default) + + results = [ + _mk_track_result(username='badpeer', filename='albums/bad.flac'), + _mk_track_result(username='goodpeer', filename='albums/good.flac'), + ] + + kept = configured_client._drop_quarantined_sources(results) + + assert len(kept) == 1 + assert kept[0].username == 'goodpeer' + + +def test_drop_quarantined_sources_returns_input_when_quarantine_missing(configured_client, tmp_path, monkeypatch): + """No quarantine directory yet (fresh install / never used) — + helper returns an empty set; filter returns the input unchanged. + Defaults to today's behaviour for users with no quarantine history.""" + import core.soulseek_client as sc + monkeypatch.setattr(sc.config_manager, 'get', + lambda key, default=None: str(tmp_path) if key == 'soulseek.download_path' else default) + + results = [_mk_track_result(username='peer', filename='song.flac')] + + kept = configured_client._drop_quarantined_sources(results) + + assert kept == results + + +def test_drop_quarantined_sources_swallows_filesystem_errors(configured_client, monkeypatch): + """If something goes wrong loading the quarantine keys (permissions, + OS quirk, etc.), the filter must NOT break the download pipeline. + Returns input unchanged so legitimate downloads keep working — + same defensive contract as the existing 401/connection handlers.""" + import core.soulseek_client as sc + + def _broken_get(key, default=None): + raise RuntimeError("config explosion") + + monkeypatch.setattr(sc.config_manager, 'get', _broken_get) + + results = [_mk_track_result(username='peer', filename='song.flac')] + + kept = configured_client._drop_quarantined_sources(results) + + assert kept == results + + +def test_filter_results_by_quality_runs_quarantine_dedup_first(configured_client, tmp_path, monkeypatch): + """Integration pin: `filter_results_by_quality_preference` calls + the quarantine dedup BEFORE the quality picker. If a candidate is + on the quarantine record, it can't win the picker by virtue of + superior bitrate — that's how the #652 loop manifested.""" + import json as _json + quarantine_dir = tmp_path / 'ss_quarantine' + quarantine_dir.mkdir() + + sidecar = { + "context": { + "original_search_result": { + "username": "badpeer", "filename": "high_bitrate_bad.flac", + }, + }, + } + (quarantine_dir / "20260518_120000.json").write_text(_json.dumps(sidecar)) + + import core.soulseek_client as sc + monkeypatch.setattr(sc.config_manager, 'get', + lambda key, default=None: str(tmp_path) if key == 'soulseek.download_path' else default) + + # Mock the DB call inside filter_results_by_quality_preference so the + # test doesn't need a real DB. Quality profile permits FLAC. + class _FakeDB: + def get_quality_profile(self): + return { + 'preset': 'flac', + 'qualities': { + 'flac': {'enabled': True, 'min_kbps': 800, 'max_kbps': 99999}, + 'mp3_320': {'enabled': False, 'min_kbps': 0, 'max_kbps': 0}, + 'mp3_256': {'enabled': False, 'min_kbps': 0, 'max_kbps': 0}, + 'mp3_192': {'enabled': False, 'min_kbps': 0, 'max_kbps': 0}, + }, + 'priority': ['flac'], + } + + import database.music_database as md + monkeypatch.setattr(md, 'MusicDatabase', lambda: _FakeDB()) + + results = [ + # The "bad" source has the BEST quality on paper — pre-fix would win the picker. + _mk_track_result(username='badpeer', filename='high_bitrate_bad.flac', + quality='flac', bitrate=1411, size=20_000_000, duration=180_000), + _mk_track_result(username='goodpeer', filename='good.flac', + quality='flac', bitrate=1411, size=20_000_000, duration=180_000), + ] + + kept = configured_client.filter_results_by_quality_preference(results) + + usernames = {r.username for r in kept} + assert 'badpeer' not in usernames, "Quarantined source must be filtered before the quality picker" + assert 'goodpeer' in usernames diff --git a/tests/downloads/test_tidal_pinning.py b/tests/downloads/test_tidal_pinning.py index 74327f37..bb4db8e9 100644 --- a/tests/downloads/test_tidal_pinning.py +++ b/tests/downloads/test_tidal_pinning.py @@ -61,6 +61,26 @@ def test_is_authenticated_false_when_session_check_login_raises(tidal_client_wit assert client.is_authenticated() is False +def test_check_device_auth_returns_helpful_message_for_non_json_tidal_response(tidal_client_with_engine): + client, _ = tidal_client_with_engine + + class FakeFuture: + def running(self): + return False + + def result(self, timeout=0): + raise ValueError("Expecting value: line 1 column 1 (char 0)") + + client._device_auth_future = FakeFuture() + client._device_auth_link = {'verification_uri': 'https://link.tidal.com', 'user_code': 'ABCD'} + + result = client.check_device_auth() + + assert result['status'] == 'error' + assert 'invalid auth response' in result['message'] + assert 'Expecting value' not in result['message'] + + # --------------------------------------------------------------------------- # download() — filename parsing + id contract # --------------------------------------------------------------------------- diff --git a/tests/imports/test_duration_tolerance_resolution.py b/tests/imports/test_duration_tolerance_resolution.py new file mode 100644 index 00000000..a5ba6386 --- /dev/null +++ b/tests/imports/test_duration_tolerance_resolution.py @@ -0,0 +1,70 @@ +from core.imports.file_integrity import _MAX_USER_TOLERANCE_S, resolve_duration_tolerance + + +def test_none_returns_none_so_caller_uses_auto_scaled_default(): + assert resolve_duration_tolerance(None) is None + + +def test_missing_or_empty_string_returns_none(): + assert resolve_duration_tolerance("") is None + assert resolve_duration_tolerance(" ") is None + + +def test_zero_returns_none_to_avoid_strict_mode_ambiguity(): + # 0 means "unset" — never strict-mode (which would fail any drift). + # Users who want strict have no use-case; users who want disabled + # set a high value (capped to _MAX_USER_TOLERANCE_S). + assert resolve_duration_tolerance(0) is None + assert resolve_duration_tolerance(0.0) is None + assert resolve_duration_tolerance("0") is None + + +def test_negative_returns_none(): + assert resolve_duration_tolerance(-1) is None + assert resolve_duration_tolerance(-3.5) is None + assert resolve_duration_tolerance("-10") is None + + +def test_positive_integer_passes_through_as_float(): + assert resolve_duration_tolerance(5) == 5.0 + assert resolve_duration_tolerance(10) == 10.0 + + +def test_positive_float_passes_through(): + assert resolve_duration_tolerance(3.5) == 3.5 + assert resolve_duration_tolerance(0.1) == 0.1 + + +def test_numeric_string_parsed(): + assert resolve_duration_tolerance("5") == 5.0 + assert resolve_duration_tolerance("3.5") == 3.5 + assert resolve_duration_tolerance("10.0") == 10.0 + + +def test_unparseable_string_returns_none(): + assert resolve_duration_tolerance("abc") is None + assert resolve_duration_tolerance("five") is None + assert resolve_duration_tolerance("3s") is None + + +def test_above_max_clamped_to_ceiling(): + assert resolve_duration_tolerance(9999) == _MAX_USER_TOLERANCE_S + assert resolve_duration_tolerance(_MAX_USER_TOLERANCE_S + 1) == _MAX_USER_TOLERANCE_S + + +def test_at_ceiling_passes_through(): + assert resolve_duration_tolerance(_MAX_USER_TOLERANCE_S) == _MAX_USER_TOLERANCE_S + + +def test_non_numeric_types_return_none(): + assert resolve_duration_tolerance([5]) is None + assert resolve_duration_tolerance({"value": 5}) is None + assert resolve_duration_tolerance(object()) is None + + +def test_bool_treated_as_int_python_semantics(): + # Python: bool is int subclass. True -> 1.0, False -> 0 -> None. + # Documented behavior, not a bug — config values won't realistically + # be booleans for a numeric setting. + assert resolve_duration_tolerance(True) == 1.0 + assert resolve_duration_tolerance(False) is None diff --git a/tests/imports/test_import_routes.py b/tests/imports/test_import_routes.py new file mode 100644 index 00000000..cfc7d8ce --- /dev/null +++ b/tests/imports/test_import_routes.py @@ -0,0 +1,600 @@ +import os +from concurrent.futures import Future + +import core.imports.routes as import_routes +from core.imports.routes import ( + ImportRouteRuntime, + album_match, + album_process, + process_single_import_file, + search_albums, + search_tracks, + singles_process, + staging_files, + staging_groups, + staging_hints, + staging_suggestions, +) + + +class _FakeLogger: + def __init__(self): + self.debug_messages = [] + self.error_messages = [] + self.info_messages = [] + self.warning_messages = [] + + def debug(self, msg, *args): + self.debug_messages.append(msg % args if args else msg) + + def error(self, msg, *args): + self.error_messages.append(msg % args if args else msg) + + def info(self, msg, *args): + self.info_messages.append(msg % args if args else msg) + + def warning(self, msg, *args): + self.warning_messages.append(msg % args if args else msg) + + +class _FakeHydrabaseWorker: + def __init__(self): + self.enqueued = [] + + def enqueue(self, query, kind): + self.enqueued.append((query, kind)) + + +class _FakeAutomationEngine: + def __init__(self, fail=False): + self.events = [] + self.fail = fail + + def emit(self, name, payload): + if self.fail: + raise RuntimeError("emit boom") + self.events.append((name, payload)) + + +class _FakeExecutor: + def __init__(self, outcomes=None): + self.outcomes = list(outcomes or []) + self.calls = [] + + def submit(self, fn, *args): + self.calls.append((fn, args)) + future = Future() + if self.outcomes: + outcome = self.outcomes.pop(0) + if isinstance(outcome, BaseException): + future.set_exception(outcome) + else: + future.set_result(outcome) + else: + future.set_result(fn(*args)) + return future + + +def _touch(path): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"") + + +def _metadata_for(files): + def _read_metadata(file_path, rel_path): + return files[rel_path] + + return _read_metadata + + +def test_staging_files_returns_audio_files_with_metadata(tmp_path): + _touch(tmp_path / "Artist" / "02 - Song.flac") + _touch(tmp_path / "cover.jpg") + rel_song = os.path.join("Artist", "02 - Song.flac") + metadata = { + rel_song: { + "title": "Song", + "artist": "Track Artist", + "albumartist": "Album Artist", + "album": "Album", + "track_number": 2, + "disc_number": 1, + } + } + runtime = ImportRouteRuntime( + get_staging_path=lambda: str(tmp_path), + read_staging_file_metadata=_metadata_for(metadata), + logger=_FakeLogger(), + ) + + payload, status = staging_files(runtime) + + assert status == 200 + assert payload["success"] is True + assert payload["staging_path"] == str(tmp_path) + assert len(payload["files"]) == 1 + assert payload["files"] == [ + { + "filename": "02 - Song.flac", + "rel_path": rel_song, + "full_path": str(tmp_path / "Artist" / "02 - Song.flac"), + "title": "Song", + "artist": "Album Artist", + "album": "Album", + "track_number": 2, + "disc_number": 1, + "extension": ".flac", + } + ] + + +def test_staging_groups_only_returns_multi_file_album_groups(tmp_path): + _touch(tmp_path / "a.mp3") + _touch(tmp_path / "b.mp3") + _touch(tmp_path / "single.mp3") + metadata = { + "a.mp3": { + "title": "A", + "artist": "Artist", + "albumartist": "", + "album": "Album", + "track_number": 2, + "disc_number": 1, + }, + "b.mp3": { + "title": "B", + "artist": "Artist", + "albumartist": "", + "album": "Album", + "track_number": 1, + "disc_number": 1, + }, + "single.mp3": { + "title": "Single", + "artist": "Other", + "albumartist": "", + "album": "Other Album", + "track_number": 1, + "disc_number": 1, + }, + } + runtime = ImportRouteRuntime( + get_staging_path=lambda: str(tmp_path), + read_staging_file_metadata=_metadata_for(metadata), + logger=_FakeLogger(), + ) + + payload, status = staging_groups(runtime) + + assert status == 200 + assert payload["success"] is True + assert len(payload["groups"]) == 1 + group = payload["groups"][0] + assert group["album"] == "Album" + assert group["artist"] == "Artist" + assert group["file_count"] == 2 + assert [f["filename"] for f in group["files"]] == ["b.mp3", "a.mp3"] + + +def test_staging_hints_prefers_tag_queries_then_folder_queries(tmp_path): + _touch(tmp_path / "Folder_Album" / "01.mp3") + _touch(tmp_path / "Folder_Album" / "02.mp3") + _touch(tmp_path / "Loose" / "track.flac") + + def _read_tags(file_path): + if file_path.endswith("01.mp3") or file_path.endswith("02.mp3"): + return {"album": ["Tagged Album"], "artist": ["Tagged Artist"]} + return {} + + runtime = ImportRouteRuntime( + get_staging_path=lambda: str(tmp_path), + read_tags=_read_tags, + logger=_FakeLogger(), + ) + + payload, status = staging_hints(runtime) + + assert status == 200 + assert payload == { + "success": True, + "hints": ["Tagged Album Tagged Artist", "Folder Album", "Loose"], + } + + +def test_staging_suggestions_returns_cache_payload(monkeypatch): + monkeypatch.setattr( + import_routes, + "get_import_suggestions_cache", + lambda: {"suggestions": [{"album": "Album"}], "built": True}, + ) + monkeypatch.setattr(import_routes, "_get_primary_source", lambda: "deezer") + + payload, status = staging_suggestions() + + assert status == 200 + assert payload == { + "success": True, + "suggestions": [{"album": "Album"}], + "ready": True, + "primary_source": "deezer", + } + + +def test_staging_groups_returns_empty_for_missing_staging_path(tmp_path): + runtime = ImportRouteRuntime( + get_staging_path=lambda: str(tmp_path / "missing"), + logger=_FakeLogger(), + ) + + payload, status = staging_groups(runtime) + + assert status == 200 + assert payload == {"success": True, "groups": []} + + +def test_staging_hints_returns_empty_for_missing_staging_path(tmp_path): + runtime = ImportRouteRuntime( + get_staging_path=lambda: str(tmp_path / "missing"), + logger=_FakeLogger(), + ) + + payload, status = staging_hints(runtime) + + assert status == 200 + assert payload == {"success": True, "hints": []} + + +def test_staging_files_returns_error_when_path_resolution_fails(): + logger = _FakeLogger() + runtime = ImportRouteRuntime( + get_staging_path=lambda: (_ for _ in ()).throw(RuntimeError("path boom")), + logger=logger, + ) + + payload, status = staging_files(runtime) + + assert status == 500 + assert payload["success"] is False + assert payload["error"] == "path boom" + assert logger.error_messages == ["Error scanning staging files: path boom"] + + +def test_staging_groups_returns_error_when_metadata_read_fails(tmp_path): + _touch(tmp_path / "a.mp3") + logger = _FakeLogger() + runtime = ImportRouteRuntime( + get_staging_path=lambda: str(tmp_path), + read_staging_file_metadata=lambda _file_path, _rel_path: (_ for _ in ()).throw(RuntimeError("tag boom")), + logger=logger, + ) + + payload, status = staging_groups(runtime) + + assert status == 500 + assert payload["success"] is False + assert payload["error"] == "tag boom" + assert logger.error_messages == ["Error building staging groups: tag boom"] + + +def test_staging_hints_returns_error_when_path_resolution_fails(): + logger = _FakeLogger() + runtime = ImportRouteRuntime( + get_staging_path=lambda: (_ for _ in ()).throw(RuntimeError("hint boom")), + logger=logger, + ) + + payload, status = staging_hints(runtime) + + assert status == 500 + assert payload["success"] is False + assert payload["error"] == "hint boom" + assert logger.error_messages == ["Error getting staging hints: hint boom"] + + +def test_search_albums_enqueues_hydrabase_and_caps_limit(): + worker = _FakeHydrabaseWorker() + calls = [] + runtime = ImportRouteRuntime( + get_primary_source=lambda: "hydrabase", + hydrabase_worker=worker, + dev_mode_enabled=True, + search_import_albums=lambda query, limit: calls.append((query, limit)) or [{"id": "album-1"}], + logger=_FakeLogger(), + ) + + payload, status = search_albums(runtime, " Album ", 99) + + assert status == 200 + assert payload == { + "success": True, + "albums": [{"id": "album-1"}], + "primary_source": "hydrabase", + } + assert worker.enqueued == [("Album", "albums")] + assert calls == [("Album", 50)] + + +def test_search_albums_requires_query(): + payload, status = search_albums(ImportRouteRuntime(logger=_FakeLogger()), "", 12) + + assert status == 400 + assert payload == {"success": False, "error": "Missing query parameter"} + + +def test_search_albums_exposes_primary_source_when_chain_falls_back(): + # Pins github issue #681: when the primary source returns nothing and the + # silent fallback chain (intentional, see core/auto_import_worker.py:1316) + # serves results from a different source, the response must carry both + # `primary_source` (what the user configured) and per-album `source` + # (what actually served the result) so the UI can warn the user. + runtime = ImportRouteRuntime( + get_primary_source=lambda: "musicbrainz", + search_import_albums=lambda query, limit: [ + {"id": "deezer-1", "name": "Album", "source": "deezer"}, + ], + logger=_FakeLogger(), + ) + + payload, status = search_albums(runtime, "Weapons of Mass Destruction", 12) + + assert status == 200 + assert payload["success"] is True + assert payload["primary_source"] == "musicbrainz" + assert payload["albums"][0]["source"] == "deezer" + + +def test_search_tracks_enqueues_hydrabase_and_caps_limit(): + worker = _FakeHydrabaseWorker() + calls = [] + runtime = ImportRouteRuntime( + get_primary_source=lambda: "hydrabase", + hydrabase_worker=worker, + dev_mode_enabled=True, + search_import_tracks=lambda query, limit: calls.append((query, limit)) or [{"id": "track-1"}], + logger=_FakeLogger(), + ) + + payload, status = search_tracks(runtime, " Track ", 99) + + assert status == 200 + assert payload == { + "success": True, + "tracks": [{"id": "track-1"}], + "primary_source": "hydrabase", + } + assert worker.enqueued == [("Track", "tracks")] + assert calls == [("Track", 30)] + + +def test_album_match_warns_without_source_and_passes_file_filter(): + logger = _FakeLogger() + calls = [] + runtime = ImportRouteRuntime( + build_album_import_match_payload=lambda *args, **kwargs: calls.append((args, kwargs)) or {"success": True, "matches": []}, + logger=logger, + ) + + payload, status = album_match( + runtime, + { + "album_id": "album-1", + "album_name": "Album", + "album_artist": "Artist", + "file_paths": ["a.flac", "b.flac"], + }, + ) + + assert status == 200 + assert payload == {"success": True, "matches": []} + assert calls == [ + ( + ("album-1",), + { + "album_name": "Album", + "album_artist": "Artist", + "file_paths": {"a.flac", "b.flac"}, + "source": None, + }, + ) + ] + assert len(logger.warning_messages) == 1 + assert "Missing 'source'" in logger.warning_messages[0] + + +def test_album_match_requires_album_id(): + payload, status = album_match(ImportRouteRuntime(logger=_FakeLogger()), {}) + + assert status == 400 + assert payload == {"success": False, "error": "Missing album_id"} + + +def test_album_process_posts_valid_files_and_records_side_effects(tmp_path): + good_file = tmp_path / "good.flac" + _touch(good_file) + processed_contexts = [] + activity = [] + refresh_calls = [] + automation = _FakeAutomationEngine() + runtime = ImportRouteRuntime( + resolve_album_artist_context=lambda album, source="": {"name": "Artist"}, + build_album_import_context=lambda album, track, **kwargs: {"album": album, "track": track, **kwargs}, + post_process_matched_download=lambda key, context, path: processed_contexts.append((key, context, path)), + add_activity_item=lambda *args: activity.append(args), + refresh_import_suggestions_cache=lambda: refresh_calls.append("refresh"), + automation_engine=automation, + logger=_FakeLogger(), + ) + + payload, status = album_process( + runtime, + { + "album": {"id": "album-1", "name": "Album", "artist": "Artist", "source": "deezer"}, + "matches": [ + { + "staging_file": {"full_path": str(good_file), "filename": "good.flac"}, + "track": {"name": "Good Track", "track_number": 1, "disc_number": 2}, + }, + { + "staging_file": {"full_path": str(tmp_path / "missing.flac"), "filename": "missing.flac"}, + "track": {"name": "Missing Track", "track_number": 2}, + }, + ], + }, + ) + + assert status == 200 + assert payload == { + "success": True, + "processed": 1, + "total": 2, + "errors": ["File not found: missing.flac"], + } + assert len(processed_contexts) == 1 + key, context, path = processed_contexts[0] + assert key.startswith("import_album_album-1_1_") + assert context["artist_context"] == {"name": "Artist"} + assert context["total_discs"] == 2 + assert context["source"] == "deezer" + assert path == str(good_file) + assert activity == [("", "Album Imported", "Album by Artist (1/2 tracks)", "Now")] + assert refresh_calls == ["refresh"] + assert automation.events == [ + ( + "import_completed", + {"track_count": "1", "album_name": "Album", "artist": "Artist"}, + ), + ( + "batch_complete", + { + "playlist_name": "Import: Album", + "total_tracks": "2", + "completed_tracks": "1", + "failed_tracks": "1", + }, + ), + ] + + +def test_album_process_requires_album_and_matches(): + payload, status = album_process(ImportRouteRuntime(logger=_FakeLogger()), {"album": {}, "matches": []}) + + assert status == 400 + assert payload == {"success": False, "error": "Missing album or matches data"} + + +def test_process_single_import_file_resolves_and_posts_context(tmp_path): + audio_file = tmp_path / "Artist - Song.flac" + _touch(audio_file) + post_calls = [] + runtime = ImportRouteRuntime( + parse_filename_metadata=lambda filename: {"title": "Song", "artist": "Artist"}, + get_single_track_import_context=lambda title, artist, **kwargs: { + "source": "deezer", + "context": {"track": {"name": title}, "artist": {"name": artist}}, + }, + normalize_import_context=lambda context: context, + get_import_context_artist=lambda context: context["artist"], + get_import_track_info=lambda context: context["track"], + post_process_matched_download=lambda key, context, path: post_calls.append((key, context, path)), + logger=_FakeLogger(), + ) + + outcome = process_single_import_file(runtime, {"full_path": str(audio_file), "filename": audio_file.name}) + + assert outcome == ("ok", "Song") + assert len(post_calls) == 1 + assert post_calls[0][0].startswith("import_single_") + assert post_calls[0][1] == {"track": {"name": "Song"}, "artist": {"name": "Artist"}} + assert post_calls[0][2] == str(audio_file) + + +def test_process_single_import_file_rejects_malformed_manual_match(tmp_path): + audio_file = tmp_path / "Song.flac" + _touch(audio_file) + runtime = ImportRouteRuntime(post_process_matched_download=lambda *_args: None, logger=_FakeLogger()) + + outcome = process_single_import_file( + runtime, + {"full_path": str(audio_file), "filename": audio_file.name, "manual_match": {"id": "track-1"}}, + ) + + assert outcome == ("error", "Malformed manual match for file: Song.flac") + + +def test_singles_process_aggregates_worker_results_and_side_effects(): + activity = [] + refresh_calls = [] + automation = _FakeAutomationEngine() + executor = _FakeExecutor(outcomes=[("ok", "Song"), ("error", "Bad Song")]) + runtime = ImportRouteRuntime( + import_singles_executor=executor, + add_activity_item=lambda *args: activity.append(args), + refresh_import_suggestions_cache=lambda: refresh_calls.append("refresh"), + automation_engine=automation, + logger=_FakeLogger(), + ) + + payload, status = singles_process( + runtime, + [{"filename": "a.flac"}, {"filename": "b.flac"}], + ) + + assert status == 200 + assert payload == { + "success": True, + "processed": 1, + "total": 2, + "errors": ["Bad Song"], + } + assert len(executor.calls) == 2 + assert activity == [("", "Singles Imported", "1/2 tracks processed", "Now")] + assert refresh_calls == ["refresh"] + assert automation.events == [ + ( + "import_completed", + {"track_count": "1", "album_name": "", "artist": "Various"}, + ), + ( + "batch_complete", + { + "playlist_name": "Import: Singles", + "total_tracks": "2", + "completed_tracks": "1", + "failed_tracks": "1", + }, + ), + ] + + +def test_singles_process_uses_injected_single_file_worker(): + calls = [] + executor = _FakeExecutor() + + def fake_process_file(runtime, file_info): + calls.append((runtime, file_info)) + return ("ok", file_info["filename"]) + + runtime = ImportRouteRuntime( + import_singles_executor=executor, + process_single_import_file=fake_process_file, + logger=_FakeLogger(), + ) + + payload, status = singles_process(runtime, [{"filename": "patched.flac"}]) + + assert status == 200 + assert payload == { + "success": True, + "processed": 1, + "total": 1, + "errors": [], + } + assert calls == [(runtime, {"filename": "patched.flac"})] + assert executor.calls[0][0] is fake_process_file + + +def test_singles_process_requires_files(): + payload, status = singles_process(ImportRouteRuntime(logger=_FakeLogger()), []) + + assert status == 400 + assert payload == {"success": False, "error": "No files provided"} diff --git a/tests/imports/test_import_side_effects.py b/tests/imports/test_import_side_effects.py index 455d2d4f..136ee765 100644 --- a/tests/imports/test_import_side_effects.py +++ b/tests/imports/test_import_side_effects.py @@ -2,6 +2,8 @@ import os import sqlite3 from types import SimpleNamespace +import pytest + from core.imports import side_effects @@ -364,6 +366,42 @@ def test_library_history_labels_auto_import(monkeypatch): assert captured["title"] == "Auto-Imported Track" +@pytest.mark.parametrize( + ("username", "expected"), + [ + ("torrent", "Torrent"), + ("usenet", "Usenet"), + ("staging", "Staging"), + ], +) +def test_library_history_labels_release_and_staging_sources(monkeypatch, username, expected): + """Release/staging imports should not fall through to the Soulseek label.""" + captured = {} + + class _DBStub: + def add_library_history_entry(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(side_effects, "get_database", lambda: _DBStub()) + + context = { + "track_info": { + "name": "Imported Track", + "artists": [{"name": "Some Artist"}], + "album": "Some Album", + }, + "original_search_result": { + "username": username, + "filename": "source-file.flac", + }, + "_final_processed_path": "/library/some-album/01.flac", + } + + side_effects.record_library_history_download(context) + + assert captured["download_source"] == expected + + # --------------------------------------------------------------------------- # Album duration parity — must equal sum of all track durations, not whatever # the first imported track happened to be. diff --git a/tests/imports/test_quarantine_management.py b/tests/imports/test_quarantine_management.py new file mode 100644 index 00000000..d2c527c0 --- /dev/null +++ b/tests/imports/test_quarantine_management.py @@ -0,0 +1,400 @@ +import json +import os + +from core.imports.quarantine import ( + approve_quarantine_entry, + delete_quarantine_entry, + entry_id_from_quarantined_filename, + get_quarantined_source_keys, + list_quarantine_entries, + recover_to_staging, + serialize_quarantine_context, +) +from core.imports.pipeline import _should_skip_quarantine_check + + +# ────────────────────────────────────────────────────────────────────── +# serialize_quarantine_context — JSON-safe coercion +# ────────────────────────────────────────────────────────────────────── + +def test_serialize_passes_scalar_dict_unchanged(): + ctx = {"title": "DNA.", "track_number": 2, "active": True, "missing": None, "duration_ms": 185000} + out = serialize_quarantine_context(ctx) + assert out == ctx + + +def test_serialize_walks_nested_dicts(): + ctx = {"track_info": {"name": "DNA.", "artists": [{"name": "Kendrick"}, {"name": "Rihanna"}]}} + out = serialize_quarantine_context(ctx) + assert out == ctx + + +def test_serialize_coerces_set_to_list(): + ctx = {"sources": {"spotify", "deezer"}} + out = serialize_quarantine_context(ctx) + assert sorted(out["sources"]) == ["deezer", "spotify"] + + +def test_serialize_coerces_tuple_to_list(): + ctx = {"pair": (1, 2, 3)} + out = serialize_quarantine_context(ctx) + assert out == {"pair": [1, 2, 3]} + + +def test_serialize_stringifies_unknown_objects(): + class Custom: + def __str__(self): + return "" + out = serialize_quarantine_context({"obj": Custom()}) + assert out["obj"] == "" + + +def test_serialize_non_dict_returns_empty_dict(): + assert serialize_quarantine_context(None) == {} + assert serialize_quarantine_context("string") == {} + assert serialize_quarantine_context([1, 2, 3]) == {} + + +def test_serialize_round_trips_through_json(): + ctx = { + "track_info": {"name": "X", "artists": [{"name": "A"}, {"name": "B"}]}, + "spotify_artist": {"name": "A", "id": "abc"}, + "duration_ms": 180000, + "sources": {"spotify"}, + } + serialized = serialize_quarantine_context(ctx) + json.dumps(serialized) # must not raise + + +# ────────────────────────────────────────────────────────────────────── +# list_quarantine_entries +# ────────────────────────────────────────────────────────────────────── + +def _write_entry(quarantine_dir, entry_id, original_name, *, with_context=False, trigger="integrity", reason="boom", file_bytes=b"X" * 100): + qfile = quarantine_dir / f"{entry_id}_{original_name}.quarantined" + qfile.write_bytes(file_bytes) + sidecar = { + "original_filename": original_name, + "quarantine_reason": reason, + "expected_track": "Track", + "expected_artist": "Artist", + "timestamp": "2026-05-14T12:00:00", + "trigger": trigger, + } + if with_context: + sidecar["context"] = {"track_info": {"name": "Track"}, "context_key": entry_id} + sidecar_path = quarantine_dir / f"{entry_id}_{os.path.splitext(original_name)[0]}.json" + sidecar_path.write_text(json.dumps(sidecar)) + return qfile, sidecar_path + + +def test_list_returns_empty_for_missing_dir(tmp_path): + assert list_quarantine_entries(str(tmp_path / "nope")) == [] + + +def test_list_returns_empty_for_empty_dir(tmp_path): + assert list_quarantine_entries(str(tmp_path)) == [] + + +def test_list_returns_entry_with_sidecar_fields(tmp_path): + _write_entry(tmp_path, "20260514_120000", "song.flac", reason="Duration mismatch") + entries = list_quarantine_entries(str(tmp_path)) + assert len(entries) == 1 + e = entries[0] + assert e["original_filename"] == "song.flac" + assert e["reason"] == "Duration mismatch" + assert e["expected_track"] == "Track" + assert e["expected_artist"] == "Artist" + assert e["has_full_context"] is False + assert e["trigger"] == "integrity" + assert e["size_bytes"] == 100 + + +def test_list_flags_full_context_entries(tmp_path): + _write_entry(tmp_path, "20260514_120000", "song.flac", with_context=True) + entries = list_quarantine_entries(str(tmp_path)) + assert entries[0]["has_full_context"] is True + + +def test_list_handles_orphan_quarantined_file_without_sidecar(tmp_path): + qfile = tmp_path / "20260514_120000_orphan.flac.quarantined" + qfile.write_bytes(b"X") + entries = list_quarantine_entries(str(tmp_path)) + assert len(entries) == 1 + assert entries[0]["reason"] == "Unknown reason" + assert entries[0]["has_full_context"] is False + + +def test_list_skips_orphan_sidecars_without_file(tmp_path): + sidecar = tmp_path / "20260514_120000_only.json" + sidecar.write_text(json.dumps({"original_filename": "only.flac", "quarantine_reason": "x"})) + assert list_quarantine_entries(str(tmp_path)) == [] + + +def test_list_sorts_newest_first(tmp_path): + _write_entry(tmp_path, "20260101_120000", "old.flac") + _write_entry(tmp_path, "20260514_120000", "new.flac") + entries = list_quarantine_entries(str(tmp_path)) + assert entries[0]["original_filename"] == "new.flac" + assert entries[1]["original_filename"] == "old.flac" + + +def test_list_swallows_corrupt_sidecar_gracefully(tmp_path): + qfile = tmp_path / "20260514_120000_song.flac.quarantined" + qfile.write_bytes(b"X") + sidecar = tmp_path / "20260514_120000_song.json" + sidecar.write_text("{ this is not valid json") + entries = list_quarantine_entries(str(tmp_path)) + assert len(entries) == 1 + assert entries[0]["reason"] == "Unknown reason" + + +def test_entry_id_helper_handles_paths_and_quarantine_suffix(): + path = "/music/ss_quarantine/20260514_120000_song.flac.quarantined" + assert entry_id_from_quarantined_filename(path) == "20260514_120000_song" + + +def test_quarantine_bypass_all_skips_every_gate(): + context = {"_skip_quarantine_check": "all"} + assert _should_skip_quarantine_check(context, "integrity") is True + assert _should_skip_quarantine_check(context, "acoustid") is True + assert _should_skip_quarantine_check(context, "bit_depth") is True + + +# ────────────────────────────────────────────────────────────────────── +# delete_quarantine_entry +# ────────────────────────────────────────────────────────────────────── + +def test_delete_removes_both_file_and_sidecar(tmp_path): + _write_entry(tmp_path, "20260514_120000", "song.flac") + assert delete_quarantine_entry(str(tmp_path), "20260514_120000_song") is True + assert not (tmp_path / "20260514_120000_song.flac.quarantined").exists() + assert not (tmp_path / "20260514_120000_song.json").exists() + + +def test_delete_returns_false_when_entry_missing(tmp_path): + assert delete_quarantine_entry(str(tmp_path), "nonexistent") is False + + +def test_delete_handles_orphan_file_without_sidecar(tmp_path): + qfile = tmp_path / "20260514_120000_orphan.flac.quarantined" + qfile.write_bytes(b"X") + assert delete_quarantine_entry(str(tmp_path), "20260514_120000_orphan") is True + assert not qfile.exists() + + +# ────────────────────────────────────────────────────────────────────── +# approve_quarantine_entry — full-context path +# ────────────────────────────────────────────────────────────────────── + +def test_approve_restores_file_and_returns_context_and_trigger(tmp_path): + quarantine = tmp_path / "ss_quarantine" + quarantine.mkdir() + restore = tmp_path / "restore" + + _write_entry(quarantine, "20260514_120000", "song.flac", with_context=True, trigger="integrity") + + result = approve_quarantine_entry(str(quarantine), "20260514_120000_song", str(restore)) + assert result is not None + restored_path, context, trigger = result + assert os.path.basename(restored_path) == "song.flac" + assert os.path.isfile(restored_path) + assert context["track_info"]["name"] == "Track" + assert trigger == "integrity" + # Sidecar removed after approve + assert not (quarantine / "20260514_120000_song.json").exists() + + +def test_approve_returns_none_for_thin_sidecar_without_context(tmp_path): + _write_entry(tmp_path, "20260514_120000", "song.flac", with_context=False) + result = approve_quarantine_entry(str(tmp_path), "20260514_120000_song", str(tmp_path / "restore")) + assert result is None + + +def test_approve_returns_none_for_missing_entry(tmp_path): + assert approve_quarantine_entry(str(tmp_path), "nope", str(tmp_path)) is None + + +def test_approve_avoids_filename_collision(tmp_path): + quarantine = tmp_path / "q" + quarantine.mkdir() + restore = tmp_path / "r" + restore.mkdir() + (restore / "song.flac").write_bytes(b"existing") + _write_entry(quarantine, "20260514_120000", "song.flac", with_context=True) + result = approve_quarantine_entry(str(quarantine), "20260514_120000_song", str(restore)) + assert result is not None + restored_path = result[0] + assert os.path.basename(restored_path) == "song_(2).flac" + assert (restore / "song.flac").read_bytes() == b"existing" + + +# ────────────────────────────────────────────────────────────────────── +# recover_to_staging — fallback for thin sidecars +# ────────────────────────────────────────────────────────────────────── + +def test_recover_strips_prefix_and_suffix(tmp_path): + quarantine = tmp_path / "q" + quarantine.mkdir() + staging = tmp_path / "s" + + qfile, _ = _write_entry(quarantine, "20260514_120000", "song.flac") + + target = recover_to_staging(str(quarantine), str(staging), "20260514_120000_song") + assert target is not None + assert os.path.basename(target) == "song.flac" + assert os.path.isfile(target) + assert not qfile.exists() + + +def test_recover_uses_sidecar_original_filename_when_available(tmp_path): + quarantine = tmp_path / "q" + quarantine.mkdir() + staging = tmp_path / "s" + qfile = quarantine / "20260514_120000_munged_name.flac.quarantined" + qfile.write_bytes(b"X") + sidecar = quarantine / "20260514_120000_munged_name.json" + sidecar.write_text(json.dumps({"original_filename": "Pretty Track Name.flac"})) + + target = recover_to_staging(str(quarantine), str(staging), "20260514_120000_munged_name") + assert target is not None + assert os.path.basename(target) == "Pretty Track Name.flac" + + +def test_recover_returns_none_for_missing_entry(tmp_path): + assert recover_to_staging(str(tmp_path / "q"), str(tmp_path / "s"), "nope") is None + + +def test_recover_avoids_filename_collision(tmp_path): + quarantine = tmp_path / "q" + quarantine.mkdir() + staging = tmp_path / "s" + staging.mkdir() + (staging / "song.flac").write_bytes(b"existing") + _write_entry(quarantine, "20260514_120000", "song.flac") + + target = recover_to_staging(str(quarantine), str(staging), "20260514_120000_song") + assert target is not None + assert os.path.basename(target) == "song_(2).flac" + + +def test_recover_removes_sidecar_after_move(tmp_path): + quarantine = tmp_path / "q" + quarantine.mkdir() + staging = tmp_path / "s" + _, sidecar = _write_entry(quarantine, "20260514_120000", "song.flac") + + recover_to_staging(str(quarantine), str(staging), "20260514_120000_song") + assert not sidecar.exists() + + +# ────────────────────────────────────────────────────────────────────── +# get_quarantined_source_keys — issue #652 dedup primitive +# ────────────────────────────────────────────────────────────────────── + + +def _write_quarantine_sidecar_with_source(quarantine_dir, entry_id, *, + username=None, filename=None): + """Helper that writes a sidecar matching the shape `move_to_quarantine` + produces — `context.original_search_result.{username, filename}` is + the path `get_quarantined_source_keys` pulls from.""" + sidecar = { + "original_filename": "song.flac", + "quarantine_reason": "boom", + "timestamp": "2026-05-14T12:00:00", + "trigger": "acoustid", + } + if username is not None or filename is not None: + sidecar["context"] = { + "original_search_result": { + "username": username or "", + "filename": filename or "", + } + } + path = quarantine_dir / f"{entry_id}.json" + path.write_text(json.dumps(sidecar)) + return path + + +def test_source_keys_empty_for_missing_dir(tmp_path): + """Defensive: caller may pass a path that doesn't exist (config not + initialised, quarantine never used). Don't crash, just return an + empty set — Soulseek filter then keeps every candidate.""" + assert get_quarantined_source_keys(str(tmp_path / "nope")) == set() + + +def test_source_keys_empty_for_empty_dir(tmp_path): + """Empty quarantine dir → empty set.""" + assert get_quarantined_source_keys(str(tmp_path)) == set() + + +def test_source_keys_collects_username_filename_tuples(tmp_path): + """Sidecars with `context.original_search_result.username` and + `.filename` round-trip into `(username, filename)` tuples — that's + the exact shape the Soulseek candidate filter looks up against.""" + _write_quarantine_sidecar_with_source( + tmp_path, "20260514_120000_a", + username="badpeer", filename="path/to/bad.flac", + ) + _write_quarantine_sidecar_with_source( + tmp_path, "20260514_120100_b", + username="otherpeer", filename="other.mp3", + ) + + keys = get_quarantined_source_keys(str(tmp_path)) + + assert ("badpeer", "path/to/bad.flac") in keys + assert ("otherpeer", "other.mp3") in keys + assert len(keys) == 2 + + +def test_source_keys_skip_legacy_sidecars_without_context(tmp_path): + """Sidecars written pre-Feb 2026 don't have the `context` field — + can't gate against them since the originating source is unknown. + Must skip silently rather than crashing the dedup path.""" + _write_quarantine_sidecar_with_source(tmp_path, "legacy_id") # no username/filename + + assert get_quarantined_source_keys(str(tmp_path)) == set() + + +def test_source_keys_skip_sidecars_with_empty_source_fields(tmp_path): + """Defensive: a sidecar with an empty string for username OR filename + can't gate anything meaningfully — dropping every result whose + username equals '' would catch unrelated downloads. Skip those + entries entirely.""" + _write_quarantine_sidecar_with_source(tmp_path, "empty_user", username="", filename="x.flac") + _write_quarantine_sidecar_with_source(tmp_path, "empty_file", username="u", filename="") + + assert get_quarantined_source_keys(str(tmp_path)) == set() + + +def test_source_keys_skip_corrupt_sidecars(tmp_path): + """A corrupt JSON sidecar (truncated write, encoding glitch) must + not propagate up and break the dedup path. Filesystem read errors + are swallowed at debug level.""" + bad = tmp_path / "corrupt.json" + bad.write_text("{not valid json") + _write_quarantine_sidecar_with_source( + tmp_path, "good", username="good_peer", filename="good.flac", + ) + + keys = get_quarantined_source_keys(str(tmp_path)) + + assert keys == {("good_peer", "good.flac")} + + +def test_source_keys_dedup_repeated_sources(tmp_path): + """If the SAME `(username, filename)` was quarantined twice (which + is exactly the #652 bug — but until now wasn't being prevented), + the set collapses to one entry. The Soulseek filter still acts as + a single-membership check, so a single set entry is enough.""" + _write_quarantine_sidecar_with_source( + tmp_path, "first", username="peer", filename="dupe.flac", + ) + _write_quarantine_sidecar_with_source( + tmp_path, "second", username="peer", filename="dupe.flac", + ) + + keys = get_quarantined_source_keys(str(tmp_path)) + + assert keys == {("peer", "dupe.flac")} diff --git a/tests/library/test_missing_track_import.py b/tests/library/test_missing_track_import.py new file mode 100644 index 00000000..077810be --- /dev/null +++ b/tests/library/test_missing_track_import.py @@ -0,0 +1,260 @@ +"""Tests for the Enhanced Library "I Have This" import service.""" + +from __future__ import annotations + +import os +import shutil +import sqlite3 +from dataclasses import dataclass + +import pytest + +from core.library import missing_track_import as mti + + +class _ConnCtx: + def __init__(self, conn): + self.conn = conn + + def __enter__(self): + return self.conn + + def __exit__(self, exc_type, exc, tb): + return False + + +class _FakeDB: + def __init__(self, conn): + self.conn = conn + + def _get_connection(self): + return _ConnCtx(self.conn) + + +@dataclass +class _FakeConfig: + download_path: str + active_server: str = "navidrome" + + def get(self, key, default=None): + if key == "soulseek.download_path": + return self.download_path + return default + + def get_active_media_server(self): + return self.active_server + + +def _make_db(*, include_disc_number: bool = True) -> tuple[_FakeDB, sqlite3.Connection]: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + cur = conn.cursor() + cur.execute( + """ + CREATE TABLE artists ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL + ) + """ + ) + cur.execute( + """ + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id TEXT NOT NULL, + title TEXT NOT NULL, + year INTEGER, + track_count INTEGER, + server_source TEXT, + deezer_id TEXT, + thumb_url TEXT + ) + """ + ) + disc_col = ", disc_number INTEGER DEFAULT 1" if include_disc_number else "" + cur.execute( + f""" + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT NOT NULL, + artist_id TEXT NOT NULL, + title TEXT NOT NULL, + track_number INTEGER{disc_col}, + duration INTEGER, + file_path TEXT, + bitrate INTEGER, + file_size INTEGER, + server_source TEXT, + deezer_id TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + cur.execute("INSERT INTO artists (id, name) VALUES ('artist-1', 'Kendrick Lamar')") + cur.execute( + """ + INSERT INTO albums (id, artist_id, title, year, track_count, server_source, deezer_id) + VALUES ('album-basic', 'artist-1', 'DAMN.', 2017, 14, 'navidrome', '302127') + """ + ) + cur.execute( + """ + INSERT INTO albums (id, artist_id, title, year, track_count, server_source, deezer_id) + VALUES ('album-deluxe', 'artist-1', 'DAMN. COLLECTORS EDITION', 2017, 14, 'navidrome', '999999') + """ + ) + conn.commit() + return _FakeDB(conn), conn + + +def _insert_track(conn, *, track_id, album_id, title, track_number, file_path, disc_number=1): + columns = [row[1] for row in conn.execute("PRAGMA table_info(tracks)").fetchall()] + if "disc_number" in columns: + conn.execute( + """ + INSERT INTO tracks (id, album_id, artist_id, title, track_number, disc_number, duration, file_path, bitrate, file_size, server_source) + VALUES (?, ?, 'artist-1', ?, ?, ?, 177000, ?, 900, 1234, 'navidrome') + """, + (track_id, album_id, title, track_number, disc_number, str(file_path)), + ) + else: + conn.execute( + """ + INSERT INTO tracks (id, album_id, artist_id, title, track_number, duration, file_path, bitrate, file_size, server_source) + VALUES (?, ?, 'artist-1', ?, ?, 177000, ?, 900, 1234, 'navidrome') + """, + (track_id, album_id, title, track_number, str(file_path)), + ) + conn.commit() + + +def _deps(tmp_path, db, *, post_process_fn=None, sync_calls=None): + sync_calls = sync_calls if sync_calls is not None else [] + + def _default_post_process(_key, context, staged_path): + final_dir = tmp_path / "Library" / "Kendrick Lamar - 2017 DAMN" + final_dir.mkdir(parents=True, exist_ok=True) + final_path = final_dir / "08 - HUMBLE [FLAC 16bit].flac" + shutil.copy2(staged_path, final_path) + context["_final_processed_path"] = str(final_path) + + return mti.MissingTrackImportDeps( + database=db, + config_manager=_FakeConfig(str(tmp_path / "downloads")), + post_process_fn=post_process_fn or _default_post_process, + resolve_library_file_path_fn=lambda path: str(path) if path and os.path.exists(path) else None, + docker_resolve_path_fn=lambda path: path, + sync_tracks_to_server_fn=lambda rows, server: sync_calls.append((rows, server)), + service_id_columns={"deezer": {"track": "deezer_id"}}, + ) + + +def _payload(): + return { + "source_track_id": "deluxe-humble", + "album_source_id": "302127", + "total_discs": 1, + "expected_track": { + "title": "HUMBLE.", + "track_number": 8, + "disc_number": 1, + "duration": 177000, + "source": "deezer", + "track_id": "350171311", + "deezer_id": "350171311", + "artists": ["Kendrick Lamar"], + }, + } + + +def test_import_existing_track_copies_file_and_writes_target_album_row(tmp_path, monkeypatch): + db, conn = _make_db(include_disc_number=True) + source_file = tmp_path / "deluxe" / "08 - HUMBLE.flac" + source_file.parent.mkdir() + source_file.write_bytes(b"source audio") + sibling_file = tmp_path / "basic" / "01 - BLOOD.flac" + sibling_file.parent.mkdir() + sibling_file.write_bytes(b"sibling audio") + _insert_track(conn, track_id="basic-blood", album_id="album-basic", title="BLOOD.", track_number=1, file_path=sibling_file) + _insert_track(conn, track_id="deluxe-humble", album_id="album-deluxe", title="HUMBLE.", track_number=8, file_path=source_file) + + inherited = [] + monkeypatch.setattr(mti, "read_album_identity_tags", lambda path: {"musicbrainz_albumid": "target-release"} if path == str(sibling_file) else {}) + monkeypatch.setattr(mti, "write_album_identity_tags", lambda path, tags: inherited.append((path, tags)) or True) + + sync_calls = [] + result = mti.import_existing_track_for_album_slot("album-basic", _payload(), _deps(tmp_path, db, sync_calls=sync_calls)) + + assert source_file.read_bytes() == b"source audio" + assert os.path.exists(result["final_path"]) + assert inherited == [(result["final_path"], {"musicbrainz_albumid": "target-release"})] + + row = conn.execute("SELECT * FROM tracks WHERE album_id = 'album-basic' AND track_number = 8").fetchone() + assert row is not None + assert row["title"] == "HUMBLE." + assert row["disc_number"] == 1 + assert row["file_path"] == result["final_path"] + assert row["deezer_id"] == "350171311" + assert sync_calls and sync_calls[0][1] == "navidrome" + + +def test_import_adds_disc_number_column_for_older_track_tables(tmp_path, monkeypatch): + db, conn = _make_db(include_disc_number=False) + source_file = tmp_path / "deluxe" / "08 - HUMBLE.flac" + source_file.parent.mkdir() + source_file.write_bytes(b"source audio") + sibling_file = tmp_path / "basic" / "01 - BLOOD.flac" + sibling_file.parent.mkdir() + sibling_file.write_bytes(b"sibling audio") + _insert_track(conn, track_id="basic-blood", album_id="album-basic", title="BLOOD.", track_number=1, file_path=sibling_file) + _insert_track(conn, track_id="deluxe-humble", album_id="album-deluxe", title="HUMBLE.", track_number=8, file_path=source_file) + + write_calls = [] + monkeypatch.setattr(mti, "read_album_identity_tags", lambda path: {"musicbrainz_albumid": "target-release"} if path == str(sibling_file) else {}) + monkeypatch.setattr(mti, "write_album_identity_tags", lambda path, tags: write_calls.append((path, tags)) or True) + + result = mti.import_existing_track_for_album_slot("album-basic", _payload(), _deps(tmp_path, db)) + + columns = [row[1] for row in conn.execute("PRAGMA table_info(tracks)").fetchall()] + assert "disc_number" in columns + row = conn.execute("SELECT title, disc_number, file_path FROM tracks WHERE album_id = 'album-basic' AND track_number = 8").fetchone() + assert row["title"] == "HUMBLE." + assert row["disc_number"] == 1 + assert row["file_path"] == result["final_path"] + assert write_calls, "album identity inheritance should still run after old DB migration" + + +def test_copy_album_identity_uses_target_sibling_and_leaves_track_tags_to_imported_file(tmp_path, monkeypatch): + db, conn = _make_db(include_disc_number=True) + sibling_file = tmp_path / "basic" / "01 - BLOOD.flac" + sibling_file.parent.mkdir() + sibling_file.write_bytes(b"sibling") + final_file = tmp_path / "basic" / "08 - HUMBLE.flac" + final_file.write_bytes(b"imported") + _insert_track(conn, track_id="basic-blood", album_id="album-basic", title="BLOOD.", track_number=1, file_path=sibling_file) + + monkeypatch.setattr(mti, "read_album_identity_tags", lambda path: {"musicbrainz_albumid": "target-release", "barcode": "target-barcode"}) + writes = [] + monkeypatch.setattr(mti, "write_album_identity_tags", lambda path, tags: writes.append((path, tags)) or True) + + copied = mti.copy_album_identity_from_target_sibling( + db, + "album-basic", + str(final_file), + 1, + 8, + lambda path: str(path) if os.path.exists(path) else None, + ) + + assert copied is True + assert writes == [(str(final_file), {"musicbrainz_albumid": "target-release", "barcode": "target-barcode"})] + + +def test_import_rejects_missing_expected_track_context(tmp_path): + db, _conn = _make_db(include_disc_number=True) + with pytest.raises(mti.MissingTrackImportError) as exc: + mti.import_existing_track_for_album_slot("album-basic", {"source_track_id": "x", "expected_track": {}}, _deps(tmp_path, db)) + + assert exc.value.status_code == 400 + assert "expected_track" in str(exc.value) diff --git a/tests/matching/test_acoustid_candidates.py b/tests/matching/test_acoustid_candidates.py new file mode 100644 index 00000000..81d2bbce --- /dev/null +++ b/tests/matching/test_acoustid_candidates.py @@ -0,0 +1,201 @@ +"""Tests for the shared AcoustID candidate-matching helper. + +Issue #587 / Foxxify report — scanner used to treat ``recordings[0]`` +as authoritative, so when AcoustID returned multiple candidates and +the top one was the wrong-credited recording (different MB entry +under the same fingerprint), the scanner created a false-positive +"Wrong download" finding even though a lower-ranked candidate matched +the expected metadata exactly. +""" + +from __future__ import annotations + +from difflib import SequenceMatcher + +import pytest + +from core.matching.acoustid_candidates import ( + duration_mismatches_strongly, + find_matching_recording, +) + + +def _ratio_sim(a: str, b: str) -> float: + """Reasonable test similarity that handles non-trivial differences.""" + if not a or not b: + return 0.0 + return SequenceMatcher(None, a.lower().strip(), b.lower().strip()).ratio() + + +# ────────────────────────────────────────────────────────────────────── +# find_matching_recording +# ────────────────────────────────────────────────────────────────────── + +def test_top_recording_matches_returns_immediately(): + recordings = [ + {'title': 'Nana', 'artist': 'Geoxor'}, + {'title': 'Nana', 'artist': 'Edward Vesala Trio'}, + ] + result, t_sim, a_sim = find_matching_recording( + recordings, 'Nana', 'Geoxor', similarity=_ratio_sim, + ) + assert result == {'title': 'Nana', 'artist': 'Geoxor'} + assert t_sim == 1.0 + assert a_sim == 1.0 + + +def test_falls_through_to_lower_ranked_match_for_foxxify_nana_case(): + """Reporter case 2: top AcoustID candidate is 'Nana' by 'Edward + Vesala Trio' (97% fingerprint), but the LOWER-ranked candidate + is the expected 'Nana' by 'Geoxor'. Pre-fix scanner saw only [0] + and flagged. Post-fix returns the matching candidate.""" + recordings = [ + {'title': 'Nana', 'artist': 'Edward Vesala Trio'}, # AcoustID's top match + {'title': 'Nana', 'artist': 'Geoxor'}, # the actual right one + ] + result, _, _ = find_matching_recording( + recordings, 'Nana', 'Geoxor', similarity=_ratio_sim, + ) + assert result == {'title': 'Nana', 'artist': 'Geoxor'} + + +def test_no_match_returns_none_with_best_seen_sims(): + """When no candidate passes thresholds, return the best-seen sims + so callers can log the closest near-miss in the finding.""" + recordings = [ + {'title': 'Different Song', 'artist': 'Different Artist'}, + {'title': 'Sort Of Close', 'artist': 'Different Artist'}, + ] + result, t_sim, a_sim = find_matching_recording( + recordings, 'Different', 'AnotherArtist', + similarity=_ratio_sim, + title_threshold=0.95, + artist_threshold=0.95, + ) + assert result is None + # Best seen — even though no candidate passed the threshold + assert t_sim > 0.0 + assert a_sim > 0.0 + + +def test_skips_recordings_missing_title_or_artist(): + recordings = [ + {'title': None, 'artist': 'Geoxor'}, + {'title': 'Nana', 'artist': ''}, + {'title': 'Nana', 'artist': 'Geoxor'}, + ] + result, _, _ = find_matching_recording( + recordings, 'Nana', 'Geoxor', similarity=_ratio_sim, + ) + assert result == {'title': 'Nana', 'artist': 'Geoxor'} + + +def test_skips_non_dict_entries(): + recordings = [None, 'string', {'title': 'Nana', 'artist': 'Geoxor'}] + result, _, _ = find_matching_recording( + recordings, 'Nana', 'Geoxor', similarity=_ratio_sim, + ) + assert result == {'title': 'Nana', 'artist': 'Geoxor'} + + +def test_empty_inputs_return_none(): + assert find_matching_recording([], 'X', 'Y')[0] is None + assert find_matching_recording([{'title': 'X', 'artist': 'Y'}], '', 'Y')[0] is None + assert find_matching_recording([{'title': 'X', 'artist': 'Y'}], 'X', '')[0] is None + + +def test_separate_artist_similarity_function_is_honored(): + """Verifier passes alias-aware comparison via artist_similarity. + Make sure it's used instead of the generic similarity.""" + recordings = [{'title': 'Track', 'artist': '澤野弘之'}] + + def alias_aware(expected, actual): + # Pretend our alias chain bridges Hiroyuki Sawano ↔ 澤野弘之 + if expected == 'Hiroyuki Sawano' and actual == '澤野弘之': + return 1.0 + return 0.0 + + result, _, a_sim = find_matching_recording( + recordings, 'Track', 'Hiroyuki Sawano', + similarity=_ratio_sim, + artist_similarity=alias_aware, + ) + assert result is not None + assert a_sim == 1.0 + + +def test_skip_predicate_drops_unwanted_candidates(): + """Verifier uses skip_predicate to drop wrong-version recordings + (instrumental vs vocal, etc.).""" + recordings = [ + {'title': 'Track (Instrumental)', 'artist': 'X'}, + {'title': 'Track', 'artist': 'X'}, + ] + result, _, _ = find_matching_recording( + recordings, 'Track', 'X', + similarity=_ratio_sim, + skip_predicate=lambda r: 'instrumental' in (r.get('title') or '').lower(), + ) + assert result == {'title': 'Track', 'artist': 'X'} + + +def test_title_threshold_can_be_lowered_for_loose_matching(): + recordings = [{'title': 'Sort Of Close', 'artist': 'Right Artist'}] + # With strict default threshold this fails + result_strict, _, _ = find_matching_recording( + recordings, 'Different', 'Right Artist', similarity=_ratio_sim, + ) + assert result_strict is None + # With a permissive threshold the artist match alone wouldn't help — + # title sim must also pass. + result_loose, _, _ = find_matching_recording( + recordings, 'Different', 'Right Artist', + similarity=_ratio_sim, title_threshold=0.0, + ) + assert result_loose is not None + + +# ────────────────────────────────────────────────────────────────────── +# duration_mismatches_strongly — guard against fingerprint collisions +# ────────────────────────────────────────────────────────────────────── + +def test_durations_within_tolerance_pass(): + # 3-minute track, 1-second drift — well within tolerance + assert duration_mismatches_strongly(180, 181) is False + # 3-minute vs 4-minute — within the 60s absolute tolerance + assert duration_mismatches_strongly(180, 240) is False + + +def test_drift_above_absolute_floor_flags(): + # 3-minute expected, 5-minute candidate (120s drift > 63s threshold) + assert duration_mismatches_strongly(180, 300) is True + + +def test_relative_tolerance_scales_with_long_tracks(): + # 30-minute expected vs 12-minute candidate (1080s vs 720s) — + # 18-minute drift > 35% of 30min = 10.5min → mismatch + assert duration_mismatches_strongly(1800, 720) is True + # 30-minute expected vs 28-minute candidate — 2min drift = under + # max(60s, 35%*30min) = max(60, 630) = 630s → still safe + assert duration_mismatches_strongly(1800, 1680) is False + + +def test_reporter_17min_mashup_vs_5min_track_flagged(): + """Foxxify's 17min mashup edit vs 5min late-70s Japanese hiphop — + fingerprint collision. Duration guard should mark this suspicious.""" + assert duration_mismatches_strongly(17 * 60, 5 * 60) is True + + +def test_unknown_duration_returns_false_no_behavior_change(): + """When either side is missing duration, don't change behavior.""" + assert duration_mismatches_strongly(None, 300) is False + assert duration_mismatches_strongly(180, None) is False + assert duration_mismatches_strongly(0, 300) is False + assert duration_mismatches_strongly(180, 0) is False + assert duration_mismatches_strongly(-5, 300) is False + + +def test_string_or_int_durations_handled(): + # Defensive — coerce numeric types + assert duration_mismatches_strongly(180.5, 181.0) is False + assert duration_mismatches_strongly(int(180), int(300)) is True diff --git a/tests/matching/test_album_context_title.py b/tests/matching/test_album_context_title.py new file mode 100644 index 00000000..0c711a19 --- /dev/null +++ b/tests/matching/test_album_context_title.py @@ -0,0 +1,168 @@ +"""Tests for the album-context-aware track-title stripping helper. + +Issue #589 — MTV Unplugged track titles like ``"Shy Away (MTV Unplugged +Live)"`` got false-rejected by the album-scoped library check because +the local DB stored title is just ``"Shy Away"``. The pure helper here +strips the redundant suffix when (and only when) the album title +implies the same context. +""" + +from core.matching.album_context_title import ( + album_context_markers, + strip_redundant_album_suffix, +) + + +# ────────────────────────────────────────────────────────────────────── +# album_context_markers +# ────────────────────────────────────────────────────────────────────── + +def test_mtv_unplugged_album_carries_unplugged_marker(): + # 'mtv' isn't a version marker on its own — the unplugged token is + # the load-bearing one. Implied-live logic adds 'live' coverage too. + markers = album_context_markers('MTV Unplugged') + assert 'unplugged' in markers + + +def test_live_at_album_carries_live_marker(): + markers = album_context_markers('Live At Wembley') + assert 'live' in markers + + +def test_studio_album_has_no_markers(): + assert album_context_markers('Scorpion') == () + assert album_context_markers('DAMN.') == () + assert album_context_markers('') == () + assert album_context_markers(None) == () + + +def test_acoustic_session_album_marker(): + assert 'acoustic' in album_context_markers('Acoustic Sessions Vol. 2') + assert 'session' in album_context_markers('Acoustic Sessions Vol. 2') + + +# ────────────────────────────────────────────────────────────────────── +# strip_redundant_album_suffix — the headline cases from #589 +# ────────────────────────────────────────────────────────────────────── + +def test_strips_mtv_unplugged_live_suffix_when_album_is_mtv_unplugged(): + assert strip_redundant_album_suffix('Shy Away (MTV Unplugged Live)', 'MTV Unplugged') == 'Shy Away' + + +def test_strips_complex_mtv_unplugged_suffix_with_year(): + # Reporter case 2: "Only If For A Night (MTV Unplugged, 2012 / Live)" + assert strip_redundant_album_suffix( + 'Only If For A Night (MTV Unplugged, 2012 / Live)', + 'Ceremonials (Live At MTV Unplugged)', + ) == 'Only If For A Night' + + +def test_strips_dash_style_live_suffix_when_album_is_live(): + assert strip_redundant_album_suffix( + 'Bohemian Rhapsody - Live At Wembley', + 'Live At Wembley Stadium', + ) == 'Bohemian Rhapsody' + + +def test_strips_brackets_live_suffix(): + assert strip_redundant_album_suffix( + 'Hello [Live]', + 'Live At The Royal Albert Hall', + ) == 'Hello' + + +# ────────────────────────────────────────────────────────────────────── +# Negative cases — must NOT strip when it would mask a genuine variant +# ────────────────────────────────────────────────────────────────────── + +def test_does_not_strip_instrumental_when_album_is_studio(): + # Critical anti-regression — keeping AcoustID's vocal/instrumental + # gate working downstream. Don't drop the marker just because the + # title is on a studio album. + assert strip_redundant_album_suffix( + 'In My Feelings (Instrumental)', + 'Scorpion', + ) == 'In My Feelings (Instrumental)' + + +def test_does_not_strip_remix_when_album_is_studio(): + assert strip_redundant_album_suffix( + 'Hello (Acoustic Remix)', + 'Scorpion', + ) == 'Hello (Acoustic Remix)' + + +def test_does_not_strip_live_when_album_does_not_imply_live(): + # User's "Live At Wembley" might be a single-track release on an + # otherwise-studio album. Don't strip. + assert strip_redundant_album_suffix( + 'Hello (Live At Wembley)', + 'Greatest Hits', + ) == 'Hello (Live At Wembley)' + + +def test_does_not_strip_when_suffix_carries_extra_context(): + # Suffix has both the album marker AND a featured-artist credit; + # the credit isn't album context, so keep the suffix. + assert strip_redundant_album_suffix( + 'Track Name (Live - feat. Other Artist)', + 'Live At Wembley', + ) == 'Track Name (Live - feat. Other Artist)' + + +def test_no_suffix_returns_unchanged(): + assert strip_redundant_album_suffix('Shy Away', 'MTV Unplugged') == 'Shy Away' + + +def test_empty_or_none_inputs_handled(): + assert strip_redundant_album_suffix('', 'MTV Unplugged') == '' + assert strip_redundant_album_suffix(None, 'MTV Unplugged') == '' + assert strip_redundant_album_suffix('Shy Away', '') == 'Shy Away' + assert strip_redundant_album_suffix('Shy Away', None) == 'Shy Away' + + +# ────────────────────────────────────────────────────────────────────── +# Stacked-suffix cases +# ────────────────────────────────────────────────────────────────────── + +def test_strips_stacked_redundant_suffixes(): + # Some sources double up: parens + brackets, both album-context + assert strip_redundant_album_suffix( + 'Track Name (Live) [Unplugged]', + 'MTV Unplugged Live', + ) == 'Track Name' + + +def test_stops_stripping_when_remaining_suffix_is_genuine(): + # Outer is redundant (live → album-context), inner is not (remix) + assert strip_redundant_album_suffix( + 'Track Name (Remix) (Live)', + 'Live At Wembley', + ) == 'Track Name (Remix)' + + +# ────────────────────────────────────────────────────────────────────── +# Year + connector tolerance +# ────────────────────────────────────────────────────────────────────── + +def test_year_in_suffix_does_not_block_stripping(): + assert strip_redundant_album_suffix( + 'Track Name (Live, 2012)', + 'Live At Wembley', + ) == 'Track Name' + + +def test_version_word_in_suffix_does_not_block_stripping(): + # "Live Version" is still album-context (just the word "version" + # in there). Strip. + assert strip_redundant_album_suffix( + 'Track Name (Live Version)', + 'Live At Wembley', + ) == 'Track Name' + + +def test_session_marker_preserved_for_acoustic_session_album(): + assert strip_redundant_album_suffix( + 'Hello (Acoustic Session)', + 'Acoustic Sessions Vol. 2', + ) == 'Hello' diff --git a/tests/matching/test_artist_alias_lookup_586.py b/tests/matching/test_artist_alias_lookup_586.py new file mode 100644 index 00000000..97c9f03d --- /dev/null +++ b/tests/matching/test_artist_alias_lookup_586.py @@ -0,0 +1,243 @@ +"""Cross-script artist alias lookup — issue #586. + +The verifier's alias path was missing the Cyrillic spelling for +"Dmitry Yablonsky" because: + +1. ``fetch_artist_aliases`` only read ``data['aliases']`` and ignored + the canonical ``name`` / ``sort-name`` fields. When MB's canonical + name IS the cross-script form, the Latin spelling never made it + into the alias output (and vice-versa). + +2. ``lookup_artist_aliases`` ran search in strict mode only, which + queries ``artist:"..."`` and skips alias / sortname indexes. Cross- + script searches found nothing under strict. + +3. The trust gate weighted local similarity 70%, so cross-script + matches scored ~0.30 even when MB's own confidence was 100, getting + rejected as low-confidence. + +These tests pin all three fixes plus the original Hiroyuki Sawano +case from #442 (regression guard). +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from core.musicbrainz_service import MusicBrainzService + + +@pytest.fixture +def service(): + """Build a MusicBrainzService with mocked client + DB. The instance + is built bypassing __init__ so we don't need real config / DB.""" + svc = MusicBrainzService.__new__(MusicBrainzService) + svc.mb_client = MagicMock() + svc._calculate_similarity = lambda a, b: _simple_sim(a, b) + svc.get_artist_aliases = MagicMock(return_value=[]) + svc._check_cache = MagicMock(return_value=None) + svc._save_to_cache = MagicMock() + return svc + + +def _simple_sim(a: str, b: str) -> float: + """Tiny stub similarity — exact match=1.0 else 0.0. Cross-script + pairs naturally fall into 0.0 since no characters overlap.""" + if not a or not b: + return 0.0 + return 1.0 if a.lower() == b.lower() else 0.0 + + +# ────────────────────────────────────────────────────────────────────── +# fetch_artist_aliases — canonical name + sort-name now included +# ────────────────────────────────────────────────────────────────────── + +def test_fetch_aliases_includes_canonical_name(service): + service.mb_client.get_artist.return_value = { + 'name': 'Дмитрий Яблонский', + 'sort-name': 'Yablonsky, Dmitry', + 'aliases': [ + {'name': 'Dmitry Yablonsky', 'sort-name': 'Yablonsky, Dmitry'}, + ], + } + aliases = service.fetch_artist_aliases('mbid-yablonsky') + # Canonical name MUST be present so cross-script matching works + # whichever direction the canonical form points. + assert 'Дмитрий Яблонский' in aliases + assert 'Dmitry Yablonsky' in aliases + # Sort-name covered too + assert 'Yablonsky, Dmitry' in aliases + + +def test_fetch_aliases_dedupes_canonical_against_alias_entry(service): + # MB sometimes lists the canonical name as ALSO an alias entry. + # No duplicate output. + service.mb_client.get_artist.return_value = { + 'name': 'Hiroyuki Sawano', + 'sort-name': 'Sawano, Hiroyuki', + 'aliases': [ + {'name': 'Hiroyuki Sawano', 'sort-name': 'Sawano, Hiroyuki'}, + {'name': '澤野弘之'}, + ], + } + aliases = service.fetch_artist_aliases('mbid-sawano') + assert aliases.count('Hiroyuki Sawano') == 1 + assert '澤野弘之' in aliases + + +def test_fetch_aliases_handles_missing_canonical_gracefully(service): + service.mb_client.get_artist.return_value = { + 'aliases': [{'name': 'Dmitry Yablonsky'}], + } + aliases = service.fetch_artist_aliases('mbid-x') + assert aliases == ['Dmitry Yablonsky'] + + +def test_fetch_aliases_returns_empty_on_no_data(service): + service.mb_client.get_artist.return_value = None + assert service.fetch_artist_aliases('mbid-x') == [] + + +def test_fetch_aliases_returns_empty_on_exception(service): + service.mb_client.get_artist.side_effect = RuntimeError('boom') + assert service.fetch_artist_aliases('mbid-x') == [] + + +# ────────────────────────────────────────────────────────────────────── +# lookup_artist_aliases — strict + non-strict fallback +# ────────────────────────────────────────────────────────────────────── + +def test_lookup_falls_back_to_non_strict_when_strict_returns_nothing(service): + # Strict search returns nothing (typical cross-script case). + # Non-strict hits the alias index and finds the artist. + def search(name, limit, strict): + if strict: + return [] + return [{'id': 'mbid-yab', 'name': 'Дмитрий Яблонский', 'score': 100}] + service.mb_client.search_artist.side_effect = search + service.mb_client.get_artist.return_value = { + 'name': 'Дмитрий Яблонский', + 'aliases': [{'name': 'Dmitry Yablonsky'}], + } + + aliases = service.lookup_artist_aliases('Dmitry Yablonsky') + assert 'Дмитрий Яблонский' in aliases + assert 'Dmitry Yablonsky' in aliases + # Confirm both modes were attempted + calls = service.mb_client.search_artist.call_args_list + assert any(call.kwargs.get('strict') is True for call in calls) or any( + len(call.args) >= 3 and call.args[2] is True for call in calls + ) + + +def test_lookup_falls_back_to_non_strict_when_strict_score_too_low(service): + # Strict returns a low-confidence match (cross-script — local sim + # near 0). Non-strict hits a stronger match via alias index. + def search(name, limit, strict): + if strict: + return [{'id': 'mbid-other', 'name': 'Some Other Artist', 'score': 30}] + return [{'id': 'mbid-yab', 'name': 'Дмитрий Яблонский', 'score': 100}] + service.mb_client.search_artist.side_effect = search + service.mb_client.get_artist.return_value = { + 'name': 'Дмитрий Яблонский', + 'aliases': [{'name': 'Dmitry Yablonsky'}], + } + + aliases = service.lookup_artist_aliases('Dmitry Yablonsky') + assert 'Дмитрий Яблонский' in aliases + + +# ────────────────────────────────────────────────────────────────────── +# Trust gate — MB-score-only escape for cross-script +# ────────────────────────────────────────────────────────────────────── + +def test_trust_gate_passes_on_high_mb_score_even_with_zero_local_sim(service): + # The cross-script case: local sim ~0, MB score 100, single + # unambiguous result → should now pass. + service.mb_client.search_artist.return_value = [ + {'id': 'mbid-yab', 'name': 'Дмитрий Яблонский', 'score': 100}, + ] + service.mb_client.get_artist.return_value = { + 'name': 'Дмитрий Яблонский', + 'aliases': [{'name': 'Dmitry Yablonsky'}], + } + + aliases = service.lookup_artist_aliases('Dmitry Yablonsky') + assert 'Дмитрий Яблонский' in aliases + assert 'Dmitry Yablonsky' in aliases + + +def test_trust_gate_rejects_low_mb_score_low_local_sim(service): + # Low confidence on both axes — must NOT pull aliases (false- + # positive risk: pulling random artist's aliases). + service.mb_client.search_artist.return_value = [ + {'id': 'mbid-other', 'name': 'Some Random Artist', 'score': 40}, + ] + aliases = service.lookup_artist_aliases('Dmitry Yablonsky') + assert aliases == [] + service.mb_client.get_artist.assert_not_called() + + +def test_trust_gate_rejects_when_two_high_mb_scores_tie(service): + # Two artists named the same with score 100 → ambiguous → skip + # even with MB-only escape (the unambiguity check still gates). + service.mb_client.search_artist.return_value = [ + {'id': 'mbid-a', 'name': 'John Smith', 'score': 100}, + {'id': 'mbid-b', 'name': 'John Smith', 'score': 100}, + ] + aliases = service.lookup_artist_aliases('John Smith') + assert aliases == [] + + +def test_trust_gate_passes_combined_score_when_local_sim_strong(service): + # Same-script case from #442 — local sim high. Should still pass + # (no regression on the existing path). + service.mb_client.search_artist.return_value = [ + {'id': 'mbid-saw', 'name': 'Hiroyuki Sawano', 'score': 100}, + ] + service.mb_client.get_artist.return_value = { + 'name': 'Hiroyuki Sawano', + 'aliases': [{'name': '澤野弘之'}], + } + + aliases = service.lookup_artist_aliases('Hiroyuki Sawano') + assert '澤野弘之' in aliases + + +# ────────────────────────────────────────────────────────────────────── +# End-to-end — reporter scenario via artist_names_match +# ────────────────────────────────────────────────────────────────────── + +def test_yablonsky_reporter_scenario_end_to_end(service): + """Issue #586 exact case: expected 'Dmitry Yablonsky', actual + 'Русская филармония, Дмитрий Яблонский', MB returns artist with + canonical name in Cyrillic and Latin in aliases. Strict search + finds nothing; non-strict finds the artist with high MB score.""" + def search(name, limit, strict): + if strict: + return [] + return [{'id': 'mbid-yab', 'name': 'Дмитрий Яблонский', 'score': 100}] + service.mb_client.search_artist.side_effect = search + service.mb_client.get_artist.return_value = { + 'name': 'Дмитрий Яблонский', + 'sort-name': 'Yablonsky, Dmitry', + 'aliases': [{'name': 'Dmitry Yablonsky'}], + } + + aliases = service.lookup_artist_aliases('Dmitry Yablonsky') + + # Must include the Cyrillic canonical so artist_names_match can + # bridge the credit-split actual. + assert 'Дмитрий Яблонский' in aliases + + # Verify the full bridge with the real artist_names_match helper. + from core.matching.artist_aliases import artist_names_match + matched, score = artist_names_match( + 'Dmitry Yablonsky', + 'Русская филармония, Дмитрий Яблонский', + aliases=aliases, + ) + assert matched is True + assert score >= 0.6 diff --git a/tests/matching/test_artist_alias_service.py b/tests/matching/test_artist_alias_service.py index 2e2f1a6a..a895f942 100644 --- a/tests/matching/test_artist_alias_service.py +++ b/tests/matching/test_artist_alias_service.py @@ -81,10 +81,13 @@ class TestFetchArtistAliases: def test_extracts_alias_names_from_mb_response(self, service): """Reporter's case 1 shape: MB returns aliases for Hiroyuki Sawano including the Japanese kanji form. Extract the `name` - from each alias entry.""" + from each alias entry. Issue #586 — also include the canonical + ``name`` and ``sort-name`` from the artist record itself, plus + per-alias ``sort-name`` when present, for cross-script bridging.""" service.mb_client.get_artist.return_value = { 'id': '60d2ea34-1912-425f-bf9c-fc544e4448cd', 'name': 'Hiroyuki Sawano', + 'sort-name': 'Sawano, Hiroyuki', 'aliases': [ {'name': '澤野弘之', 'sort-name': '澤野弘之', 'locale': 'ja', 'primary': True}, {'name': 'SawanoHiroyuki', 'sort-name': 'SawanoHiroyuki', 'locale': None}, @@ -94,10 +97,11 @@ class TestFetchArtistAliases: aliases = service.fetch_artist_aliases('60d2ea34-1912-425f-bf9c-fc544e4448cd') + assert 'Hiroyuki Sawano' in aliases # canonical name + assert 'Sawano, Hiroyuki' in aliases # canonical sort-name (also matches alias sort-name) assert '澤野弘之' in aliases assert 'SawanoHiroyuki' in aliases assert 'Sawano Hiroyuki' in aliases - assert len(aliases) == 3 def test_dedup_case_insensitive(self, service): """Same name with different casing should collapse — MB @@ -125,14 +129,15 @@ class TestFetchArtistAliases: aliases = service.fetch_artist_aliases('mbid-x') assert aliases == ['Real Name'] - def test_missing_aliases_key_returns_empty(self, service): - """MB artist record might not have any aliases. Returns [] - not raises.""" + def test_missing_aliases_key_returns_canonical_name_only(self, service): + """MB artist record without an aliases array still returns the + canonical name (post-#586). Pre-fix this returned [] which + meant cross-script bridging was impossible.""" service.mb_client.get_artist.return_value = { 'id': 'mbid-x', 'name': 'Some Artist', } - assert service.fetch_artist_aliases('mbid-x') == [] + assert service.fetch_artist_aliases('mbid-x') == ['Some Artist'] def test_aliases_null_returns_empty(self, service): """MB sometimes returns `aliases: null` instead of empty array.""" @@ -476,14 +481,19 @@ class TestLookupArtistAliasesMultiTier: assert aliases == [] def test_no_search_results_returns_empty(self, service): - """Artist not found on MB — empty return, cached so we - don't re-search the same name forever.""" + """Artist not found on MB under either strict or non-strict + search — empty return, cached so we don't re-search the same + name forever. Issue #586: strict-then-non-strict means TWO + search calls per uncached lookup; the empty cache prevents + further calls on the next invocation.""" service.mb_client.search_artist.return_value = [] aliases = service.lookup_artist_aliases('NeverHeardOf') assert aliases == [] - # Second call should hit cache, not re-search + # First lookup: strict + non-strict fallback = 2 calls. + assert service.mb_client.search_artist.call_count == 2 + # Second call should hit cache, not re-search at all. service.lookup_artist_aliases('NeverHeardOf') - assert service.mb_client.search_artist.call_count == 1 + assert service.mb_client.search_artist.call_count == 2 def test_low_confidence_match_skipped(self, service): """Search returned something but the name similarity is too diff --git a/tests/matching/test_version_mismatch.py b/tests/matching/test_version_mismatch.py new file mode 100644 index 00000000..3b81f788 --- /dev/null +++ b/tests/matching/test_version_mismatch.py @@ -0,0 +1,231 @@ +"""Boundary tests for ``core.matching.version_mismatch``. + +Pin every shape the version-mismatch escape valve has to handle so +future drift fails here instead of at runtime against a real download: +one-sided bare cases (the live-recording MB-metadata-gap that issue +#607 reported), two-sided real mismatches (live vs remix — keep +strict), high vs low fingerprint score gates, title/artist threshold +gates, defensive paths. +""" + +from __future__ import annotations + +import pytest + +from core.matching.version_mismatch import is_acceptable_version_mismatch + + +class TestEqualVersions: + def test_same_version_trivially_accepted(self): + # Equal version strings — no mismatch to decide. True. + assert is_acceptable_version_mismatch( + 'live', 'live', + fingerprint_score=0.0, + title_similarity=0.0, + artist_similarity=0.0, + ) is True + + def test_both_original_accepted(self): + assert is_acceptable_version_mismatch( + 'original', 'original', + fingerprint_score=0.0, + title_similarity=0.0, + artist_similarity=0.0, + ) is True + + +class TestOneSidedBareMismatch: + """Issue #607 example 2: expected has annotation, AcoustID's MB + record is bare. Accept when fingerprint + bare titles + artist + all line up.""" + + def test_live_vs_original_high_confidence_accepted(self): + # Reporter's exact case: "Clarity (Live at ...)" vs "Clarity" + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.95, + artist_similarity=1.0, + ) is True + + def test_original_vs_live_high_confidence_accepted(self): + # Same case in the other direction. + assert is_acceptable_version_mismatch( + 'original', 'live', + fingerprint_score=0.95, + title_similarity=0.95, + artist_similarity=1.0, + ) is True + + def test_live_at_thresholds_accepted(self): + # Exactly at the thresholds for the live-aware case. + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.85, + title_similarity=0.70, + artist_similarity=0.60, + ) is True + + +class TestNonLiveOneSidedMismatchStaysStrict: + """Other version markers (instrumental / remix / acoustic / etc) + have distinct fingerprints AND MB always annotates them in the + recording title. When AcoustID returns one of these for a bare + expected (or vice versa), the file genuinely IS that version — + the user asked for the wrong cut. Reject regardless of how high + the supporting scores are. + + This narrowness is what keeps the existing + test_acoustid_version_mismatch suite passing — instrumental + vs vocal etc. stays a real mismatch.""" + + def test_remix_vs_original_rejected_at_high_confidence(self): + assert is_acceptable_version_mismatch( + 'remix', 'original', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_instrumental_vs_original_rejected_at_high_confidence(self): + # The exact case test_acoustid_version_mismatch.py: + # test_instrumental_returned_for_vocal_request_fails pins — + # vocal asked, instrumental returned, must FAIL. + assert is_acceptable_version_mismatch( + 'instrumental', 'original', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_original_vs_instrumental_rejected_at_high_confidence(self): + # Reverse direction: caller asked for vocal, file is + # instrumental. + assert is_acceptable_version_mismatch( + 'original', 'instrumental', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_acoustic_vs_original_rejected_at_high_confidence(self): + assert is_acceptable_version_mismatch( + 'acoustic', 'original', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_demo_vs_original_rejected(self): + assert is_acceptable_version_mismatch( + 'demo', 'original', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + +class TestTwoSidedMismatchStaysStrict: + """Both sides have version markers but they disagree. Real + different-recording mismatch — must reject regardless of how + high the other scores are.""" + + def test_live_vs_remix_rejected_even_at_max(self): + assert is_acceptable_version_mismatch( + 'live', 'remix', + fingerprint_score=1.0, + title_similarity=1.0, + artist_similarity=1.0, + ) is False + + def test_acoustic_vs_instrumental_rejected(self): + assert is_acceptable_version_mismatch( + 'acoustic', 'instrumental', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_live_vs_acoustic_rejected(self): + assert is_acceptable_version_mismatch( + 'live', 'acoustic', + fingerprint_score=0.95, + title_similarity=0.90, + artist_similarity=1.0, + ) is False + + +class TestThresholdGates: + """One-sided + bare but one of the supporting signals is too weak. + Reject — fall through to FAIL.""" + + def test_low_fingerprint_score_rejected(self): + # Fingerprint score below threshold. Don't trust it enough. + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.50, + title_similarity=0.95, + artist_similarity=1.0, + ) is False + + def test_low_title_similarity_rejected(self): + # Bare titles disagree → different songs, not just MB metadata gap. + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.30, + artist_similarity=1.0, + ) is False + + def test_low_artist_similarity_rejected(self): + # Wrong artist — definitely not the same recording. + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.95, + artist_similarity=0.20, + ) is False + + def test_just_below_score_threshold_rejected(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.849, # default threshold 0.85 + title_similarity=0.95, + artist_similarity=1.0, + ) is False + + def test_just_below_title_threshold_rejected(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.699, # default threshold 0.70 + artist_similarity=1.0, + ) is False + + def test_just_below_artist_threshold_rejected(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.95, + artist_similarity=0.599, # default threshold 0.60 + ) is False + + +class TestCustomThresholds: + def test_custom_score_threshold_accepts_when_loosened(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.70, + title_similarity=0.95, + artist_similarity=1.0, + score_threshold=0.65, + ) is True + + def test_custom_score_threshold_rejects_when_tightened(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.90, + title_similarity=0.95, + artist_similarity=1.0, + score_threshold=0.95, + ) is False diff --git a/tests/metadata/test_artist_resolution.py b/tests/metadata/test_artist_resolution.py new file mode 100644 index 00000000..95eb8d3a --- /dev/null +++ b/tests/metadata/test_artist_resolution.py @@ -0,0 +1,79 @@ +from core.metadata.artist_resolution import resolve_track_artists + + +def test_prefers_original_search_artists_when_populated(): + original = {"artists": [{"name": "Kendrick Lamar"}, {"name": "Rihanna"}]} + track = {"artists": [{"name": "Should Not Be Used"}]} + artist = {"name": "Primary"} + + assert resolve_track_artists(original, track, artist) == ["Kendrick Lamar", "Rihanna"] + + +def test_falls_back_to_track_info_artists_when_original_lacks_list(): + # Soulseek context shape: original_search_result has 'artist' (string) + # and no 'artists' (list). Full Spotify track object lives on track_info. + original = {"artist": "Kendrick Lamar"} + track = {"artists": [{"name": "Kendrick Lamar"}, {"name": "Rihanna"}]} + artist = {"name": "Kendrick Lamar"} + + assert resolve_track_artists(original, track, artist) == ["Kendrick Lamar", "Rihanna"] + + +def test_falls_back_to_artist_dict_name_when_no_lists_available(): + original = {"artist": "Solo Artist"} + track = {"name": "Track Title"} + artist = {"name": "Solo Artist"} + + assert resolve_track_artists(original, track, artist) == ["Solo Artist"] + + +def test_returns_empty_list_when_everything_missing(): + assert resolve_track_artists(None, None, None) == [] + assert resolve_track_artists({}, {}, {}) == [] + + +def test_handles_bare_string_artist_items(): + original = {"artists": ["Kendrick Lamar", "Rihanna"]} + assert resolve_track_artists(original, None, None) == ["Kendrick Lamar", "Rihanna"] + + +def test_mixed_dict_and_string_items_normalized(): + original = {"artists": [{"name": "Kendrick Lamar"}, "Rihanna"]} + assert resolve_track_artists(original, None, None) == ["Kendrick Lamar", "Rihanna"] + + +def test_strips_whitespace_and_drops_empty_entries(): + original = {"artists": [{"name": " Kendrick "}, {"name": ""}, " ", "Rihanna"]} + assert resolve_track_artists(original, None, None) == ["Kendrick", "Rihanna"] + + +def test_dict_item_without_name_key_skipped(): + original = {"artists": [{"id": "abc"}, {"name": "Rihanna"}]} + assert resolve_track_artists(original, None, None) == ["Rihanna"] + + +def test_non_list_artists_value_falls_through(): + original = {"artists": "Kendrick Lamar"} # string, not list + track = {"artists": [{"name": "Kendrick Lamar"}, {"name": "Rihanna"}]} + assert resolve_track_artists(original, track, None) == ["Kendrick Lamar", "Rihanna"] + + +def test_empty_original_artists_list_falls_through_to_track_info(): + original = {"artists": []} + track = {"artists": [{"name": "Kendrick Lamar"}, {"name": "Rihanna"}]} + assert resolve_track_artists(original, track, None) == ["Kendrick Lamar", "Rihanna"] + + +def test_artist_dict_name_blank_returns_empty(): + assert resolve_track_artists({}, {}, {"name": " "}) == [] + assert resolve_track_artists({}, {}, {"name": ""}) == [] + + +def test_non_string_artist_items_coerced_to_string(): + original = {"artists": [123, {"name": "Real Artist"}]} + assert resolve_track_artists(original, None, None) == ["123", "Real Artist"] + + +def test_none_artist_items_dropped(): + original = {"artists": [None, {"name": "Real Artist"}, None]} + assert resolve_track_artists(original, None, None) == ["Real Artist"] diff --git a/tests/metadata/test_artist_source_detail.py b/tests/metadata/test_artist_source_detail.py index 5b21769f..a1a1c2c8 100644 --- a/tests/metadata/test_artist_source_detail.py +++ b/tests/metadata/test_artist_source_detail.py @@ -147,6 +147,7 @@ class TestPerSourceEnrichment: def test_spotify_extracts_genres_followers_and_image_fallback(self, _stub_metadata): spotify = SimpleNamespace( get_artist=lambda aid, allow_fallback=False: { + "name": "Artist", "genres": ["alt rock", "emo"], "followers": {"total": 12345}, "images": [{"url": "https://sp/img.jpg"}], @@ -160,6 +161,26 @@ class TestPerSourceEnrichment: # image_url falls back to Spotify's image when metadata returned None assert payload["artist"]["image_url"] == "https://sp/img.jpg" + def test_empty_name_uses_source_artist_name_when_available(self, _stub_metadata): + spotify = SimpleNamespace( + get_artist=lambda aid, allow_fallback=False: { + "name": "Kendrick Lamar", + "genres": [], + "followers": {}, + "images": [], + } + ) + + payload, _ = build_source_only_artist_detail( + "2YZyLoL8N0Wb9xBt1NhZWg", "", "spotify", spotify_client=spotify, + ) + + assert payload["artist"]["name"] == "Kendrick Lamar" + assert _stub_metadata["last_discog_call"] == ( + "2YZyLoL8N0Wb9xBt1NhZWg", + "Kendrick Lamar", + ) + def test_deezer_extracts_genres_and_followers(self, _stub_metadata): deezer = SimpleNamespace( get_artist_info=lambda aid: { diff --git a/tests/metadata/test_artist_source_lookup.py b/tests/metadata/test_artist_source_lookup.py index 75ca4582..32c03cb6 100644 --- a/tests/metadata/test_artist_source_lookup.py +++ b/tests/metadata/test_artist_source_lookup.py @@ -31,6 +31,7 @@ EXPECTED_SOURCE_ID_FIELD = { "discogs": "discogs_id", "hydrabase": "soul_id", "musicbrainz": "musicbrainz_id", + "amazon": "amazon_id", } diff --git a/tests/metadata/test_cache_maintenance_retry.py b/tests/metadata/test_cache_maintenance_retry.py new file mode 100644 index 00000000..dc5ee85c --- /dev/null +++ b/tests/metadata/test_cache_maintenance_retry.py @@ -0,0 +1,44 @@ +import sqlite3 +from types import SimpleNamespace + +import core.metadata.cache as cache_module +from core.metadata.cache import MetadataCache + + +def test_maintenance_write_retries_once_after_disk_io(monkeypatch): + cache = MetadataCache() + attempts = [] + + class _Conn: + def close(self): + pass + + monkeypatch.setattr(cache, "_get_db", lambda: SimpleNamespace(_get_connection=lambda: _Conn())) + monkeypatch.setattr(cache_module.time, "sleep", lambda _seconds: None) + + def _operation(_conn): + attempts.append(1) + if len(attempts) == 1: + raise sqlite3.OperationalError("disk I/O error") + return 9 + + assert cache._run_maintenance_write("Cache eviction", _operation) == 9 + assert len(attempts) == 2 + + +def test_maintenance_write_does_not_retry_non_io_errors(monkeypatch): + cache = MetadataCache() + attempts = [] + + class _Conn: + def close(self): + pass + + monkeypatch.setattr(cache, "_get_db", lambda: SimpleNamespace(_get_connection=lambda: _Conn())) + + def _operation(_conn): + attempts.append(1) + raise sqlite3.OperationalError("syntax error") + + assert cache._run_maintenance_write("Cache eviction", _operation) == 0 + assert len(attempts) == 1 diff --git a/tests/metadata/test_deezer_track_cache_validity.py b/tests/metadata/test_deezer_track_cache_validity.py new file mode 100644 index 00000000..8ff107de --- /dev/null +++ b/tests/metadata/test_deezer_track_cache_validity.py @@ -0,0 +1,208 @@ +"""Pin the Deezer per-track cache validity check. + +Issue #588: contributors tagging worked for some tracks and not others. +Root cause was cache pollution — `/album//tracks` cached partial +records under the same key as `/track/`, and `get_track_details` +was using `track_position` alone as the "full payload" sentinel. Both +endpoints set track_position; only `/track/` sets contributors. + +These tests pin the corrected sentinel (`_is_full_track_payload`) so +the regression can't silently come back. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from core.deezer_client import _is_full_track_payload + + +# ──────────────────────────────────────────────────────────────────── +# Pure helper — payload-shape classification +# ──────────────────────────────────────────────────────────────────── + +def test_full_track_endpoint_payload_is_valid(): + payload = { + 'id': 12345, + 'title': 'Erased', + 'track_position': 1, + 'contributors': [{'name': 'Whipped Cream'}, {'name': 'Andrea Botez'}], + 'artist': {'name': 'Whipped Cream'}, + 'album': {'id': 1, 'title': 'Erased'}, + } + assert _is_full_track_payload(payload) is True + + +def test_full_track_with_empty_contributors_list_is_valid(): + # Single-artist track from /track/ still emits contributors=[] + # The KEY presence is what matters, not truthiness. + payload = { + 'id': 12345, + 'title': 'Solo Track', + 'track_position': 1, + 'contributors': [], + 'artist': {'name': 'Solo Artist'}, + } + assert _is_full_track_payload(payload) is True + + +def test_album_tracks_payload_missing_contributors_is_partial(): + # The exact shape /album//tracks returns per item — has + # track_position but no contributors. Pre-fix this passed the + # `track_position in cached` check; post-fix it correctly falls + # through to a fresh /track/ fetch. + payload = { + 'id': 12345, + 'title': 'Sacrifice', + 'track_position': 1, + 'duration': 180, + 'artist': {'name': 'Andrea Botez'}, + } + assert _is_full_track_payload(payload) is False + + +def test_search_payload_without_track_position_is_partial(): + payload = { + 'id': 12345, + 'title': 'Sacrifice', + 'artist': {'name': 'Andrea Botez'}, + 'isrc': 'XX1234567890', + } + assert _is_full_track_payload(payload) is False + + +def test_none_or_non_dict_payload_is_partial(): + assert _is_full_track_payload(None) is False + assert _is_full_track_payload([]) is False + assert _is_full_track_payload('string') is False + assert _is_full_track_payload(0) is False + + +def test_empty_dict_is_partial(): + assert _is_full_track_payload({}) is False + + +# ──────────────────────────────────────────────────────────────────── +# get_track_details — cache + fetch interaction +# ──────────────────────────────────────────────────────────────────── + +@pytest.fixture +def deezer_client(): + """Build a DeezerClient with mocked HTTP + cache. Bypasses __init__ + auth/config requirements.""" + from core.deezer_client import DeezerClient + client = DeezerClient.__new__(DeezerClient) + client._api_get = MagicMock() + return client + + +def _patch_cache(cached_payload): + """Patch the module-level cache lookup. Returns the patched cache + mock so callers can assert on store_entity calls.""" + cache = MagicMock() + cache.get_entity.return_value = cached_payload + cache.store_entity = MagicMock() + return patch('core.deezer_client.get_metadata_cache', return_value=cache), cache + + +def test_cache_hit_with_full_payload_skips_api_call(deezer_client): + full = { + 'id': 12345, + 'title': 'Erased', + 'track_position': 1, + 'contributors': [{'name': 'Whipped Cream'}, {'name': 'Andrea Botez'}], + 'artist': {'name': 'Whipped Cream'}, + 'album': {'id': 1, 'title': 'Erased', 'nb_tracks': 1}, + } + cache_patch, cache = _patch_cache(full) + with cache_patch: + result = deezer_client.get_track_details('12345') + + assert result is not None + assert result['artists'] == ['Whipped Cream', 'Andrea Botez'] + deezer_client._api_get.assert_not_called() + cache.store_entity.assert_not_called() + + +def test_cache_hit_with_partial_album_tracks_payload_refetches(deezer_client): + """The bug from #588 — partial album-tracks data should NOT be + treated as a full hit. Post-fix the client re-fetches.""" + partial = { + 'id': 12345, + 'title': 'Sacrifice', + 'track_position': 1, + 'artist': {'name': 'Andrea Botez'}, + } + fresh = { + 'id': 12345, + 'title': 'Sacrifice', + 'track_position': 1, + 'contributors': [{'name': 'Andrea Botez'}, {'name': 'Grabbitz'}], + 'artist': {'name': 'Andrea Botez'}, + 'album': {'id': 1, 'title': 'Sacrifice', 'nb_tracks': 1}, + } + cache_patch, cache = _patch_cache(partial) + deezer_client._api_get.return_value = fresh + with cache_patch: + result = deezer_client.get_track_details('12345') + + assert result is not None + assert result['artists'] == ['Andrea Botez', 'Grabbitz'] + deezer_client._api_get.assert_called_once_with('track/12345') + cache.store_entity.assert_called_once_with('deezer', 'track', '12345', fresh) + + +def test_cache_miss_fetches_fresh(deezer_client): + cache_patch, cache = _patch_cache(None) + fresh = { + 'id': 12345, + 'title': 'Sacrifice', + 'track_position': 1, + 'contributors': [{'name': 'Andrea Botez'}, {'name': 'Grabbitz'}], + 'artist': {'name': 'Andrea Botez'}, + 'album': {'id': 1, 'title': 'Sacrifice', 'nb_tracks': 1}, + } + deezer_client._api_get.return_value = fresh + with cache_patch: + result = deezer_client.get_track_details('12345') + + assert result is not None + assert result['artists'] == ['Andrea Botez', 'Grabbitz'] + deezer_client._api_get.assert_called_once_with('track/12345') + cache.store_entity.assert_called_once() + + +def test_cache_hit_with_search_shape_refetches(deezer_client): + """Search results lack track_position — same fall-through path as + partial album-tracks data.""" + search_shape = { + 'id': 12345, + 'title': 'Sacrifice', + 'artist': {'name': 'Andrea Botez'}, + 'isrc': 'XX1234567890', + } + fresh = { + 'id': 12345, + 'title': 'Sacrifice', + 'track_position': 1, + 'contributors': [{'name': 'Andrea Botez'}, {'name': 'Grabbitz'}], + 'artist': {'name': 'Andrea Botez'}, + 'album': {'id': 1, 'title': 'Sacrifice', 'nb_tracks': 1}, + } + cache_patch, _ = _patch_cache(search_shape) + deezer_client._api_get.return_value = fresh + with cache_patch: + result = deezer_client.get_track_details('12345') + + assert result is not None + assert result['artists'] == ['Andrea Botez', 'Grabbitz'] + deezer_client._api_get.assert_called_once() + + +def test_api_failure_returns_none(deezer_client): + cache_patch, _ = _patch_cache(None) + deezer_client._api_get.return_value = None + with cache_patch: + result = deezer_client.get_track_details('12345') + + assert result is None diff --git a/tests/metadata/test_image_url_normalization.py b/tests/metadata/test_image_url_normalization.py index 2832ba4c..01ba4933 100644 --- a/tests/metadata/test_image_url_normalization.py +++ b/tests/metadata/test_image_url_normalization.py @@ -10,6 +10,7 @@ from urllib.parse import quote "thumb_url", [ "/api/image-proxy?url=https%3A%2F%2Fexample.com%2Fcover.jpg", + "/api/image-cache/" + ("a" * 64), "http://host.docker.internal:4533/api/image-proxy?u=ketiska&t=abc&s=def&v=1.16.1&c=SoulSync&f=json", ], ) @@ -20,10 +21,11 @@ def test_normalize_image_url_leaves_existing_image_proxy_urls_alone(thumb_url): assert normalize_image_url(thumb_url) == thumb_url -def test_normalize_image_url_proxies_internal_http_urls(monkeypatch): - """Raw internal image URLs should still be routed through SoulSync's proxy.""" +def test_normalize_image_url_registers_internal_http_urls_with_image_cache(monkeypatch): + """Raw internal image URLs should be routed through SoulSync's hashed cache URL.""" from core.metadata import normalize_image_url from core.metadata import artwork + import core.image_cache as image_cache class _FakeConfig: def get_active_media_server(self): @@ -39,6 +41,33 @@ def test_normalize_image_url_proxies_internal_http_urls(monkeypatch): return {} monkeypatch.setattr(artwork, "get_config_manager", lambda: _FakeConfig()) + monkeypatch.setattr(image_cache, "cached_image_url", lambda u: "/api/image-cache/" + ("b" * 64)) + + url = "http://localhost:4533/cover.jpg" + assert normalize_image_url(url) == "/api/image-cache/" + ("b" * 64) + + +def test_normalize_image_url_falls_back_to_proxy_when_cache_registration_fails(monkeypatch): + """If cache registration breaks, internal URLs keep the old image-proxy behavior.""" + from core.metadata import normalize_image_url + from core.metadata import artwork + import core.image_cache as image_cache + + class _FakeConfig: + def get_active_media_server(self): + return "spotify" + + def get_plex_config(self): + return {} + + def get_jellyfin_config(self): + return {} + + def get_navidrome_config(self): + return {} + + monkeypatch.setattr(artwork, "get_config_manager", lambda: _FakeConfig()) + monkeypatch.setattr(image_cache, "cached_image_url", lambda u: (_ for _ in ()).throw(RuntimeError("boom"))) url = "http://localhost:4533/cover.jpg" assert normalize_image_url(url) == f"/api/image-proxy?url={quote(url, safe='')}" diff --git a/tests/metadata/test_metadata_discography.py b/tests/metadata/test_metadata_discography.py index 3a040f11..65ac0624 100644 --- a/tests/metadata/test_metadata_discography.py +++ b/tests/metadata/test_metadata_discography.py @@ -322,6 +322,7 @@ def test_iter_artist_discography_completion_uses_primary_source_first(monkeypatc assert spotify.album_calls == [] assert itunes.album_calls == [] assert db.album_calls and db.album_calls[0]["expected_track_count"] == 2 + assert db.album_calls[0]["strict_discography_match"] is True def test_iter_artist_discography_completion_respects_source_override(monkeypatch): @@ -360,6 +361,106 @@ def test_iter_artist_discography_completion_respects_source_override(monkeypatch assert spotify.album_calls == [] +def test_artist_discography_completion_uses_strict_matching_for_eps(monkeypatch): + monkeypatch.setattr(metadata_registry, "get_primary_source", lambda spotify_client_factory=None: "deezer") + monkeypatch.setattr(metadata_registry, "get_source_priority", lambda primary: [primary]) + monkeypatch.setattr(metadata_registry, "get_client_for_source", lambda source_name, **kwargs: None) + + db = _CompletionFakeDB(owned_tracks=1, expected_tracks=2) + events = list(metadata_completion.iter_artist_discography_completion_events( + { + "albums": [], + "singles": [{ + "id": "ep-1", + "name": "Original Motion Picture Soundtrack EP", + "album_type": "ep", + "total_tracks": 2, + }], + }, + artist_name="Composer One", + db=db, + )) + + assert events[1]["type"] == "single_completion" + assert db.album_calls[0]["strict_discography_match"] is True + + +def test_strict_discography_matching_rejects_distinct_soundtrack_siblings(): + db = object.__new__(MusicDatabase) + album = types.SimpleNamespace( + title="Star Wars: Episode I - The Phantom Menace (Original Motion Picture Soundtrack)", + artist_name="John Williams", + track_count=17, + ) + + confidence = db._calculate_album_confidence( + "Star Wars: Episode II - Attack of the Clones (Original Motion Picture Soundtrack)", + "John Williams", + album, + expected_track_count=13, + strict_discography_match=True, + ) + + assert confidence == 0.0 + + +def test_strict_discography_matching_allows_same_soundtrack_title(): + db = object.__new__(MusicDatabase) + album = types.SimpleNamespace( + title="Star Wars: Episode I - The Phantom Menace (Original Motion Picture Soundtrack)", + artist_name="John Williams", + track_count=17, + ) + + confidence = db._calculate_album_confidence( + "Star Wars: Episode I - The Phantom Menace (Original Motion Picture Soundtrack)", + "John Williams", + album, + expected_track_count=17, + strict_discography_match=True, + ) + + assert confidence >= 0.9 + + +def test_non_strict_album_matching_keeps_edition_behavior(): + db = object.__new__(MusicDatabase) + album = types.SimpleNamespace( + title="DAMN. (Deluxe Edition)", + artist_name="Kendrick Lamar", + track_count=14, + ) + + confidence = db._calculate_album_confidence( + "DAMN.", + "Kendrick Lamar", + album, + expected_track_count=14, + strict_discography_match=False, + ) + + assert confidence >= 0.9 + + +def test_strict_discography_matching_does_not_change_normal_albums(): + db = object.__new__(MusicDatabase) + album = types.SimpleNamespace( + title="DAMN. (Deluxe Edition)", + artist_name="Kendrick Lamar", + track_count=14, + ) + + confidence = db._calculate_album_confidence( + "DAMN.", + "Kendrick Lamar", + album, + expected_track_count=14, + strict_discography_match=True, + ) + + assert confidence >= 0.9 + + def test_iter_artist_discography_completion_uses_release_artist_metadata(monkeypatch): source = _FakeSourceClient() clients = {"deezer": source} diff --git a/tests/metadata/test_metadata_enrichment.py b/tests/metadata/test_metadata_enrichment.py index 3255e8c1..b1230ef7 100644 --- a/tests/metadata/test_metadata_enrichment.py +++ b/tests/metadata/test_metadata_enrichment.py @@ -188,9 +188,54 @@ def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_ assert metadata["track_number"] == 3 assert metadata["total_tracks"] == 12 assert metadata["disc_number"] == 2 + assert metadata["date"] == "2024-01-02" assert metadata["album_art_url"] == "https://img.example/album.jpg" +@pytest.mark.parametrize( + ("raw_release_date", "expected_tag_date"), + [ + ("2024-05-06", "2024-05-06"), + ("2024-05", "2024-05"), + ("2024", "2024"), + ("2024-05-06T07:08:09Z", "2024-05-06"), + ], +) +def test_extract_source_metadata_preserves_available_release_date_precision(monkeypatch, raw_release_date, expected_tag_date): + monkeypatch.setattr(ms, "get_config_manager", lambda: _Config({"file_organization.collab_artist_mode": "first"})) + + context = { + "source": "spotify", + "artist": {"name": "Artist One", "id": "123", "genres": []}, + "album": { + "name": "Album One", + "total_tracks": 12, + "release_date": raw_release_date, + }, + "track_info": { + "artists": [{"name": "Artist One"}], + "_source": "spotify", + "track_number": 3, + "disc_number": 1, + }, + "original_search_result": { + "title": "Song One", + "artists": [{"name": "Artist One"}], + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + }, + } + + metadata = me.extract_source_metadata( + context, + context["artist"], + {"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 1}, + ) + + assert metadata["date"] == expected_tag_date + + def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch): audio = _FakeAudio() symbols = _fake_symbols(audio) diff --git a/tests/metadata/test_metadata_registry.py b/tests/metadata/test_metadata_registry.py index cc83af52..9860c7fc 100644 --- a/tests/metadata/test_metadata_registry.py +++ b/tests/metadata/test_metadata_registry.py @@ -20,6 +20,16 @@ def test_metadata_source_label_maps_known_sources(): assert registry.get_metadata_source_label("deezer") == "Deezer" assert registry.get_metadata_source_label("discogs") == "Discogs" assert registry.get_metadata_source_label("hydrabase") == "Hydrabase" + assert registry.get_metadata_source_label("musicbrainz") == "MusicBrainz" + + +def test_musicbrainz_is_first_class_metadata_client(): + registry.clear_cached_metadata_clients() + client = object() + assert registry.get_client_for_source( + "musicbrainz", + musicbrainz_client_factory=lambda: client, + ) is client def test_metadata_source_label_falls_back_to_unmapped(): diff --git a/tests/metadata/test_multi_artist_tag_settings.py b/tests/metadata/test_multi_artist_tag_settings.py index 550e170d..d3acf452 100644 --- a/tests/metadata/test_multi_artist_tag_settings.py +++ b/tests/metadata/test_multi_artist_tag_settings.py @@ -99,8 +99,10 @@ class TestArtistsListPopulated: assert meta.get("_artists_list") == ["Solo Artist"] def test_no_artists_falls_through(self): - """When search response has no artists list, falls through to - the single-artist branch — no `_artists_list` written.""" + """When search response has no artists list, the artist-resolution + helper falls back to the artist_dict.name as a single-element list. + Multi-value tag write still no-ops because len([primary]) == 1. + """ from core.metadata import source as src_module context = { @@ -109,7 +111,31 @@ class TestArtistsListPopulated: } with patch.object(src_module, "get_config_manager", return_value=_make_cfg()): meta = src_module.extract_source_metadata(context, {"name": "X"}, {}) - assert "_artists_list" not in meta or meta.get("_artists_list") in (None, []) + assert meta.get("_artists_list") == ["X"] + + def test_soulseek_shape_falls_back_to_track_info_artists(self): + """Soulseek matched-download regression: original_search_result + carries 'artist' (singular string) but no 'artists' list, while + track_info (the matched Spotify track object) carries the full + multi-artist array. Helper should pull from track_info. + """ + from core.metadata import source as src_module + + context = { + "original_search_result": { + "title": "DNA.", + "artist": "Kendrick Lamar", + }, + "track_info": { + "name": "DNA.", + "artists": [{"name": "Kendrick Lamar"}, {"name": "Rihanna"}], + }, + "source": "spotify", + } + with patch.object(src_module, "get_config_manager", return_value=_make_cfg()): + meta = src_module.extract_source_metadata(context, {"name": "Kendrick Lamar"}, {}) + assert meta.get("_artists_list") == ["Kendrick Lamar", "Rihanna"] + assert meta.get("artist") == "Kendrick Lamar, Rihanna" # --------------------------------------------------------------------------- diff --git a/tests/metadata/test_musicbrainz_search.py b/tests/metadata/test_musicbrainz_search.py index 719f02ae..cec77648 100644 --- a/tests/metadata/test_musicbrainz_search.py +++ b/tests/metadata/test_musicbrainz_search.py @@ -680,6 +680,61 @@ def test_search_albums_bare_artist_no_hint_no_filter(): assert 'Revolver' in titles +# --------------------------------------------------------------------------- +# Issue #650 — 'Other' primary-type release-groups must surface +# --------------------------------------------------------------------------- + + +def test_search_albums_browse_filter_requests_other_primary_type(): + """Issue #650: pre-fix the MB browse filter requested only + `album|ep|single`, dropping every primary-type=`Other` release-group + at the API layer. For artists like Vocaloid producers and JP indie + acts whose music videos / one-off web releases are tagged Other, + that hid legitimate tracks. Pin that the filter now includes + 'other' so those release-groups round-trip into the discography.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.search_artist.return_value = [_mk_artist('Inabakumori', 'mb-i', score=100)] + client._client.browse_artist_release_groups.return_value = [] + + client.search_albums('inabakumori', limit=10) + + # Inspect the actual call args — the API filter is the lever that + # decides whether MB returns Other-typed groups at all. + args, kwargs = client._client.browse_artist_release_groups.call_args + requested_types = kwargs.get('release_types') or (args[1] if len(args) > 1 else None) + assert requested_types is not None, \ + "browse_artist_release_groups must receive an explicit release_types filter" + assert 'other' in requested_types, \ + f"'other' must be in the requested types so #650 Other-typed releases surface; got {requested_types}" + + +def test_search_albums_other_type_release_groups_appear_as_singles(): + """When MB returns an Other-typed release-group (music video, + one-off web release), it must arrive in the discography as an + Album dataclass with album_type='single' — so the downstream + binner in `core/metadata/discography.py` routes it to the Singles + section rather than burying it among LPs.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.search_artist.return_value = [_mk_artist('Inabakumori', 'mb-i', score=100)] + client._client.browse_artist_release_groups.return_value = [ + {'id': 'rg-mv', 'title': 'ロストアンブレラ', 'primary-type': 'Other', + 'first-release-date': '2018-02-27', 'secondary-types': []}, + {'id': 'rg-single', 'title': 'ラグトレイン', 'primary-type': 'Single', + 'first-release-date': '2020-01-01', 'secondary-types': []}, + ] + + albums = client.search_albums('inabakumori', limit=10) + + by_id = {a.id: a for a in albums} + assert 'rg-mv' in by_id, "Other-typed release-group must survive the filter and arrive in the result" + assert by_id['rg-mv'].album_type == 'single', \ + "Other-typed release-group must map to album_type='single' so it lands in the Singles section" + # Pre-existing single behaviour unchanged. + assert by_id['rg-single'].album_type == 'single' + + def test_recording_to_track_total_tracks_matches_media_count(): """Regression: total_tracks was initialized at 1 and summed with media track-counts, producing an off-by-one. An 11-track album reported 12.""" @@ -765,3 +820,257 @@ def test_search_tracks_text_path_filters_by_score(): titles = [t.name for t in tracks] assert 'Good' in titles assert 'Bad' not in titles + + +# --------------------------------------------------------------------------- +# get_recording_flat — Fix-popup MBID paste adapter +# --------------------------------------------------------------------------- + +def test_get_recording_flat_happy_path(): + """Recording with a release returns flat shape with album + image.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_recording.return_value = { + 'id': 'rec-abc', + 'title': 'Army of Me', + 'length': 234000, + 'artist-credit': [{'artist': {'name': 'Björk'}}], + 'releases': [{ + 'id': 'rel-xyz', + 'title': 'Post', + 'date': '1995-06-13', + 'status': 'Official', + 'media': [{'track-count': 11}], + 'release-group': {'id': 'rg-post', 'primary-type': 'Album', 'secondary-types': []}, + }], + } + + track = client.get_recording_flat('rec-abc') + + assert track is not None + assert track['id'] == 'rec-abc' + assert track['name'] == 'Army of Me' + assert track['artists'] == ['Björk'] # flat list of strings, not Spotify-shaped objects + assert track['album'] == 'Post' # flat string, not nested dict + assert track['duration_ms'] == 234000 + assert track['image_url'] # CAA URL present + assert 'musicbrainz.org/recording/rec-abc' in track['external_urls']['musicbrainz'] + + +def test_get_recording_flat_missing_mbid_returns_none(): + """No MBID → no API call, returns None.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + + assert client.get_recording_flat('') is None + assert client.get_recording_flat(None) is None + client._client.get_recording.assert_not_called() + + +def test_get_recording_flat_mb_returns_no_recording(): + """MB returns None (404 / missing) → adapter returns None.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_recording.return_value = None + + assert client.get_recording_flat('rec-missing') is None + + +def test_get_recording_flat_recording_without_release(): + """Standalone recording (no releases) — album stays empty, + image_url empty, but the rest of the shape is intact.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_recording.return_value = { + 'id': 'rec-standalone', + 'title': 'Untitled Demo', + 'length': 120000, + 'artist-credit': [{'artist': {'name': 'Unknown'}}], + 'releases': [], + } + + track = client.get_recording_flat('rec-standalone') + + assert track is not None + assert track['name'] == 'Untitled Demo' + assert track['album'] == '' + assert track['image_url'] == '' + assert track['artists'] == ['Unknown'] + assert track['duration_ms'] == 120000 + + +def test_get_recording_flat_multi_artist_credit(): + """Recording with multiple credited artists — all flatten to list of strings.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_recording.return_value = { + 'id': 'rec-collab', + 'title': 'Collab Track', + 'length': 180000, + 'artist-credit': [ + {'artist': {'name': 'Artist A'}}, + {'artist': {'name': 'Artist B'}}, + ], + 'releases': [], + } + + track = client.get_recording_flat('rec-collab') + + assert track['artists'] == ['Artist A', 'Artist B'] + + +def test_get_recording_flat_includes_match_get_track_details(): + """Sanity: passes the same includes list so the API call is cacheable + against the same key as get_track_details (one network request can + serve both surfaces if MB ever adds response caching upstream).""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_recording.return_value = None + + client.get_recording_flat('rec-x') + + client._client.get_recording.assert_called_once_with( + 'rec-x', includes=['releases', 'artist-credits', 'release-groups'] + ) + + +def test_get_recording_flat_swallows_client_errors(): + """MB client raising must not propagate to the route handler — return + None so the endpoint can render a friendly 404 instead of 500.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_recording.side_effect = RuntimeError('boom') + + assert client.get_recording_flat('rec-err') is None + + +# --------------------------------------------------------------------------- +# search_tracks_with_artist — Fix-popup cascade adapter +# --------------------------------------------------------------------------- + +def test_search_tracks_with_artist_uses_bare_query_mode(): + """The Fix-popup cascade needs MB's bare-query mode so diacritics and + bracketed suffixes don't kill recall. The adapter must pass strict=False + through to the underlying search_recording call.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.search_recording.return_value = [ + {'id': 'rec-1', 'title': 'Army of Me', 'score': 95, + 'releases': [{'id': 'rel-1', 'title': 'Post', 'date': '1995'}], + 'artist-credit': [{'name': 'Björk'}]}, + ] + + tracks = client.search_tracks_with_artist('Army of Me', 'Björk', limit=10) + + # strict=False is the critical bit — fuzzy recall, not phrase precision + client._client.search_recording.assert_called_once_with( + 'Army of Me', artist_name='Björk', limit=10, strict=False + ) + assert len(tracks) == 1 + assert tracks[0].name == 'Army of Me' + assert 'Björk' in tracks[0].artists + + +def test_search_tracks_with_artist_handles_missing_artist(): + """Track-only query (no artist) still works — empty string becomes + None, and the underlying client searches recordings without an + artist filter.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.search_recording.return_value = [ + {'id': 'rec-1', 'title': 'Some Song', 'score': 90, + 'releases': [], 'artist-credit': [{'name': 'Unknown'}]}, + ] + + client.search_tracks_with_artist('Some Song', '', limit=5) + + # Empty artist → None passed to the client so MB drops the AND clause + client._client.search_recording.assert_called_once_with( + 'Some Song', artist_name=None, limit=5, strict=False + ) + + +def test_search_tracks_with_artist_empty_returns_empty_list(): + """No track and no artist → return [] without hitting the network.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + + assert client.search_tracks_with_artist('', '', limit=10) == [] + client._client.search_recording.assert_not_called() + + +def test_search_tracks_with_artist_keeps_low_score_for_rerank(): + """Cascade path uses a low score floor (20) so MB recordings whose + title doesn't literally contain the artist name still enter the + candidate pool — the endpoint's rerank pass surfaces them by + artist-match relevance. Real example: "Army of Me" + "Bjork" — the + canonical Björk recording scores 28 in MB (title doesn't contain + "Bjork"), while title-collision covers like "Army of Me (Bjork)" + score 73-100. Strict 80 floor drops the right answer.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.search_recording.return_value = [ + {'id': 'rec-cover', 'title': 'Army of Me (Bjork)', 'score': 100, + 'releases': [], 'artist-credit': [{'name': 'HIRS Collective'}]}, + {'id': 'rec-canonical', 'title': 'Army of Me', 'score': 28, + 'releases': [], 'artist-credit': [{'name': 'Björk'}]}, + {'id': 'rec-noise', 'title': 'Bjork', 'score': 5, + 'releases': [], 'artist-credit': [{'name': 'Random'}]}, + ] + + tracks = client.search_tracks_with_artist('Army of Me', 'Bjork', limit=50) + + ids = [t.id for t in tracks] + # Score=28 canonical Björk recording is kept — the endpoint's rerank + # will surface it by artist match. + assert 'rec-canonical' in ids + assert 'rec-cover' in ids + # Score=5 is below the 20 floor — true garbage still filtered out. + assert 'rec-noise' not in ids + + +def test_search_tracks_text_keeps_min_score_default_80_for_enhanced_search(): + """The enhanced search tab path keeps the historical 80 floor because + it has no downstream rerank — unfiltered MB results would be noisy + for free-text user search.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.search_recording.return_value = [ + {'id': 'rec-good', 'title': 'Good', 'score': 95, + 'releases': [], 'artist-credit': [{'name': 'A'}]}, + {'id': 'rec-mid', 'title': 'Mid', 'score': 40, + 'releases': [], 'artist-credit': [{'name': 'A'}]}, + ] + + # No min_score → defaults to _MIN_SCORE (80) + tracks = client._search_tracks_text('Good', 'A', limit=10) + + titles = [t.name for t in tracks] + assert 'Good' in titles + assert 'Mid' not in titles + + +def test_search_tracks_text_strict_param_default_true(): + """Default strict=True preserves the historical behaviour of the + structured-query text-search fallback path — important so the + enrichment-style `search_tracks('Artist - Track')` flow stays on + field-scoped Lucene phrase matching as before.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.search_recording.return_value = [] + + client._search_tracks_text('Track', 'Artist', limit=10) + + client._client.search_recording.assert_called_once_with( + 'Track', artist_name='Artist', limit=10, strict=True + ) + + +def test_search_tracks_with_artist_swallows_client_errors(): + """MB client raising must not crash the endpoint — return [] so the + Fix-popup cascade falls through to the next source.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.search_recording.side_effect = RuntimeError('network down') + + assert client.search_tracks_with_artist('Track', 'Artist', limit=10) == [] diff --git a/tests/metadata/test_release_type.py b/tests/metadata/test_release_type.py new file mode 100644 index 00000000..9b6be12a --- /dev/null +++ b/tests/metadata/test_release_type.py @@ -0,0 +1,140 @@ +"""Tests for the canonical release-type mapper. + +Covers issue #650 — MusicBrainz's `Other` and `Broadcast` primary +types previously defaulted to `album_type='album'`, hiding music +videos and one-off releases from artist discography views. The mapper +now routes them to `single` so they land in the Singles bucket of the +artist detail page. + +Also pins the existing mappings (album/ep/single/compilation) so the +refactor of three sibling type-mappers into one shared helper doesn't +drift the historical behaviour. +""" + +from __future__ import annotations + +import pytest + +from core.metadata.release_type import map_release_group_type + + +# --------------------------------------------------------------------------- +# Pin existing primary-type mappings (no regression from refactor) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("primary_type,expected", [ + ("album", "album"), + ("Album", "album"), # MB returns title-cased values + ("ALBUM", "album"), + ("single", "single"), + ("Single", "single"), + ("ep", "ep"), + ("EP", "ep"), + ("compilation", "compilation"), + ("Compilation", "compilation"), +]) +def test_known_primary_types_map_canonically(primary_type, expected): + """Pin: case-insensitive primary-type mapping for the four + canonical types every consumer relied on pre-refactor.""" + assert map_release_group_type(primary_type) == expected + + +# --------------------------------------------------------------------------- +# Issue #650 — 'Other' and 'Broadcast' primary types +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("primary_type", ["other", "Other", "OTHER"]) +def test_other_primary_type_routes_to_singles(primary_type): + """Issue #650: MB tags music videos and one-off web releases with + `primary-type=Other`. They're functionally single-track releases, + so route them to `single` (lands in Singles section). Pre-fix + they fell through to the `album` default — placed in Albums view + where they cluttered the LP list AND, paired with the API filter, + were sometimes dropped from the discography entirely.""" + assert map_release_group_type(primary_type) == "single" + + +@pytest.mark.parametrize("primary_type", ["broadcast", "Broadcast"]) +def test_broadcast_primary_type_routes_to_singles(primary_type): + """Broadcasts (radio sessions, one-off live single transmissions) + are also single-track in practice. Same routing as 'Other'.""" + assert map_release_group_type(primary_type) == "single" + + +# --------------------------------------------------------------------------- +# Secondary-type compilation handling +# --------------------------------------------------------------------------- + + +def test_compilation_secondary_type_overrides_album_primary(): + """MB's canonical compilation pattern is `primary=Album, + secondary=[Compilation]`. The compilation secondary check must + fire even when the primary is Album, so 'Greatest Hits' style + releases land in the compilation bucket.""" + assert map_release_group_type("Album", ["Compilation"]) == "compilation" + + +def test_compilation_secondary_type_case_insensitive(): + """Secondary-type matching tolerates case + whitespace variations + in the provider response.""" + assert map_release_group_type("Album", ["compilation"]) == "compilation" + assert map_release_group_type("Album", [" Compilation "]) == "compilation" + + +def test_other_secondary_types_do_not_override_primary(): + """Only 'compilation' is checked as a secondary-type override. + Other MB secondary types (Live, Remix, Soundtrack, etc.) belong + to the discography filter at the search-adapter layer, not the + type mapper.""" + assert map_release_group_type("Album", ["Live"]) == "album" + assert map_release_group_type("Single", ["Remix"]) == "single" + + +def test_compilation_secondary_overrides_other_primary(): + """An 'Other' release tagged as Compilation lands in compilation, + not singles — secondary-type compilation is the strongest + classification signal.""" + assert map_release_group_type("Other", ["Compilation"]) == "compilation" + + +# --------------------------------------------------------------------------- +# Empty / unknown / defensive +# --------------------------------------------------------------------------- + + +def test_empty_primary_type_defaults_to_album(): + """Pin: empty / None primary-type still defaults to 'album' so + consumers that build records without complete provider data don't + suddenly land in a different bucket.""" + assert map_release_group_type("") == "album" + assert map_release_group_type(None) == "album" + + +def test_unknown_primary_type_defaults_to_album(): + """Pin: a primary-type value we don't know about defaults to + 'album'. Matches the pre-refactor fall-through so new MB + vocabulary doesn't accidentally cause a behaviour shift.""" + assert map_release_group_type("audiobook") == "album" + assert map_release_group_type("video") == "album" + + +def test_secondary_types_none_is_safe(): + """Pin: omitting secondary_types (legacy types.py call site) still + works — None and missing-arg both treated as no-secondary-types.""" + assert map_release_group_type("Album", None) == "album" + assert map_release_group_type("Album") == "album" + + +def test_secondary_types_with_none_entries_skipped(): + """Defensive: provider responses occasionally include None or empty + string in the secondary-types list. The mapper must not crash.""" + assert map_release_group_type("Album", [None, "", "Compilation"]) == "compilation" + assert map_release_group_type("Album", [None, ""]) == "album" + + +def test_whitespace_in_primary_type_normalized(): + """Defensive: a stray-whitespace primary-type still classifies.""" + assert map_release_group_type(" single ") == "single" + assert map_release_group_type(" Other ") == "single" diff --git a/tests/metadata/test_track_number_format.py b/tests/metadata/test_track_number_format.py new file mode 100644 index 00000000..8debd71e --- /dev/null +++ b/tests/metadata/test_track_number_format.py @@ -0,0 +1,103 @@ +"""Tests for the track-number tag formatter. + +Discord report (Netti93): album tracks tagged as "6/0" instead of +"6/13" when source data lacked total_tracks. Helper now returns just +"6" when total is 0 / unknown, matching what the retag tool already +did and what the ID3 spec allows. +""" + +from core.metadata.track_number_format import ( + format_track_number_tag, + format_track_number_tuple, +) + + +# ────────────────────────────────────────────────────────────────────── +# format_track_number_tag — string output for ID3 / Vorbis +# ────────────────────────────────────────────────────────────────────── + +def test_track_with_known_total_returns_slash_format(): + assert format_track_number_tag(6, 13) == "6/13" + assert format_track_number_tag(1, 1) == "1/1" + assert format_track_number_tag(99, 100) == "99/100" + + +def test_zero_total_returns_track_number_only(): + """The Netti93 case — total_tracks=0 means unknown, NOT + 'track 6 of 0'. Drop the slash.""" + assert format_track_number_tag(6, 0) == "6" + assert format_track_number_tag(1, 0) == "1" + + +def test_none_total_returns_track_number_only(): + assert format_track_number_tag(6, None) == "6" + + +def test_none_track_number_defaults_to_one(): + assert format_track_number_tag(None, 13) == "1/13" + assert format_track_number_tag(None, None) == "1" + + +def test_zero_track_number_defaults_to_one(): + """Track 0 isn't valid in any convention — coerce to 1.""" + # Note: 0 is non-negative so falls into the default-0 path which + # the formatter then treats as "default" via the explicit default + # arg. Since 0 is technically valid output of `int(0)`, the helper + # passes it through. Document the behavior here. + # Actually re-checking: 0 satisfies `>= 0` so returns 0. That + # means format would emit "0/13" for malformed input. Not great + # but at least it doesn't crash. Test pins current behavior. + assert format_track_number_tag(0, 13) == "0/13" + + +def test_negative_total_treated_as_unknown(): + assert format_track_number_tag(6, -1) == "6" + + +def test_negative_track_number_falls_back_to_default(): + assert format_track_number_tag(-1, 13) == "1/13" + + +def test_string_inputs_coerced(): + assert format_track_number_tag("6", "13") == "6/13" + assert format_track_number_tag("6", "0") == "6" + + +def test_unparseable_inputs_use_defaults(): + assert format_track_number_tag("six", "thirteen") == "1" + assert format_track_number_tag("abc", 13) == "1/13" + + +def test_float_inputs_truncate(): + # int() truncates floats — keeps behavior deterministic + assert format_track_number_tag(6.7, 13.9) == "6/13" + + +# ────────────────────────────────────────────────────────────────────── +# format_track_number_tuple — MP4 trkn tuple +# ────────────────────────────────────────────────────────────────────── + +def test_tuple_with_known_total(): + assert format_track_number_tuple(6, 13) == (6, 13) + + +def test_tuple_with_zero_total(): + assert format_track_number_tuple(6, 0) == (6, 0) + + +def test_tuple_with_none_total(): + assert format_track_number_tuple(6, None) == (6, 0) + + +def test_tuple_with_none_track_defaults_to_one(): + assert format_track_number_tuple(None, 13) == (1, 13) + assert format_track_number_tuple(None, None) == (1, 0) + + +def test_tuple_negative_inputs_safe(): + assert format_track_number_tuple(-1, -5) == (1, 0) + + +def test_tuple_string_inputs_coerced(): + assert format_track_number_tuple("6", "13") == (6, 13) + assert format_track_number_tuple("6", "0") == (6, 0) diff --git a/tests/metadata/test_typed_metadata_types.py b/tests/metadata/test_typed_metadata_types.py index 20478f75..2006636d 100644 --- a/tests/metadata/test_typed_metadata_types.py +++ b/tests/metadata/test_typed_metadata_types.py @@ -342,6 +342,31 @@ def test_album_from_musicbrainz_dict_release_group_type_overrides_default(): assert Album.from_musicbrainz_dict(raw).album_type == 'single' +def test_album_from_musicbrainz_dict_accepts_adapter_shape(): + raw = { + 'id': 'rg-or-release-mbid', + 'name': 'Coffee Break', + 'artists': [{'id': 'artist-mbid', 'name': 'Zeds Dead'}], + 'release_date': '2011-07-12', + 'total_tracks': 1, + 'album_type': 'single', + 'images': [{'url': 'https://cover.example/front.jpg'}], + 'external_urls': {'musicbrainz': 'https://musicbrainz.org/release/rg-or-release-mbid'}, + } + + album = Album.from_musicbrainz_dict(raw) + + assert album.id == 'rg-or-release-mbid' + assert album.name == 'Coffee Break' + assert album.artists == ['Zeds Dead'] + assert album.artist_id == 'artist-mbid' + assert album.release_date == '2011-07-12' + assert album.total_tracks == 1 + assert album.album_type == 'single' + assert album.image_url == 'https://cover.example/front.jpg' + assert album.external_ids['musicbrainz'] == 'rg-or-release-mbid' + + # --------------------------------------------------------------------------- # Qobuz # --------------------------------------------------------------------------- diff --git a/tests/sync/__init__.py b/tests/sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/sync/test_match_overrides.py b/tests/sync/test_match_overrides.py new file mode 100644 index 00000000..4779ca54 --- /dev/null +++ b/tests/sync/test_match_overrides.py @@ -0,0 +1,193 @@ +from unittest.mock import MagicMock + +from core.sync.match_overrides import record_manual_match, resolve_match_overrides + + +# ────────────────────────────────────────────────────────────────────── +# resolve_match_overrides — pre-pair source→server from cache +# ────────────────────────────────────────────────────────────────────── + +def test_empty_inputs_return_empty_dict(): + assert resolve_match_overrides([], [], lambda _id: None) == {} + assert resolve_match_overrides([{"source_track_id": "x"}], [], lambda _id: "y") == {} + assert resolve_match_overrides([], [{"id": "y"}], lambda _id: None) == {} + + +def test_single_cache_hit_returns_pair(): + sources = [{"source_track_id": "spotify-iron-man", "name": "Iron Man - 2012 - Remaster"}] + servers = [{"id": 5001, "title": "Iron Man"}] + cache = {"spotify-iron-man": 5001} + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + assert result == {0: 0} + + +def test_multiple_overrides_resolve_correctly(): + sources = [ + {"source_track_id": "iron"}, + {"source_track_id": "para"}, + {"source_track_id": "war"}, + ] + servers = [ + {"id": 5001, "title": "Iron Man"}, + {"id": 5002, "title": "Paranoid"}, + {"id": 5003, "title": "War Pigs"}, + ] + cache = {"iron": 5001, "para": 5002, "war": 5003} + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + assert result == {0: 0, 1: 1, 2: 2} + + +def test_source_without_track_id_skipped(): + sources = [ + {"source_track_id": "iron", "name": "Iron Man"}, + {"name": "Paranoid"}, # no source_track_id (e.g. legacy / non-mirrored) + ] + servers = [{"id": 5001, "title": "Iron Man"}, {"id": 5002, "title": "Paranoid"}] + cache = {"iron": 5001} + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + assert result == {0: 0} + + +def test_cache_miss_skipped(): + sources = [{"source_track_id": "iron"}, {"source_track_id": "para"}] + servers = [{"id": 5001, "title": "Iron Man"}, {"id": 5002, "title": "Paranoid"}] + result = resolve_match_overrides(sources, servers, lambda sid: None) + assert result == {} + + +def test_stale_cache_pointing_at_missing_server_track_skipped(): + # User cached a match → file got deleted from server → server_tracks + # no longer has 5001 → don't pair, fall through to normal matching. + sources = [{"source_track_id": "iron"}] + servers = [{"id": 9999, "title": "Different Track"}] + cache = {"iron": 5001} # 5001 no longer exists + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + assert result == {} + + +def test_server_id_str_int_coercion(): + # Cache might store ints, server_tracks might have str IDs (Plex + # ratingKey is str). Helper coerces both sides to str. + sources = [{"source_track_id": "iron"}] + servers = [{"id": "5001", "title": "Iron Man"}] + cache = {"iron": 5001} # int from cache + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + assert result == {0: 0} + + +def test_two_sources_pointing_at_same_server_track_only_first_wins(): + # Defensive — UNIQUE constraint prevents this in production but + # cache_lookup is injectable so we verify the safety. + sources = [{"source_track_id": "a"}, {"source_track_id": "b"}] + servers = [{"id": 5001, "title": "Iron Man"}] + cache = {"a": 5001, "b": 5001} + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + assert result == {0: 0} + + +def test_cache_lookup_raising_treated_as_miss(): + sources = [{"source_track_id": "iron"}] + servers = [{"id": 5001, "title": "Iron Man"}] + def boom(_sid): + raise RuntimeError("db down") + result = resolve_match_overrides(sources, servers, boom) + assert result == {} + + +def test_non_dict_source_or_server_skipped(): + sources = [None, "string", {"source_track_id": "iron"}] + servers = [{"id": 5001, "title": "Iron Man"}] + cache = {"iron": 5001} + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + # source idx 2 → server idx 0 + assert result == {2: 0} + + +def test_server_without_id_skipped(): + sources = [{"source_track_id": "iron"}] + servers = [{"title": "Iron Man"}] # no id + cache = {"iron": 5001} + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + assert result == {} + + +def test_partial_cache_hits_only_pair_those(): + sources = [ + {"source_track_id": "iron"}, + {"source_track_id": "para"}, + {"source_track_id": "war"}, + ] + servers = [ + {"id": 5001, "title": "Iron Man"}, + {"id": 5002, "title": "Paranoid"}, + {"id": 5003, "title": "War Pigs"}, + ] + # Only iron + war cached, para falls through to normal matching + cache = {"iron": 5001, "war": 5003} + result = resolve_match_overrides(sources, servers, lambda sid: cache.get(sid)) + assert result == {0: 0, 2: 2} + + +# ────────────────────────────────────────────────────────────────────── +# record_manual_match — persist user-confirmed pair +# ────────────────────────────────────────────────────────────────────── + +def test_record_persists_with_confidence_one(): + db = MagicMock() + db.save_sync_match_cache.return_value = True + ok = record_manual_match( + db, + source_track_id="spotify-iron-man", + server_source="plex", + server_track_id=5001, + server_track_title="Iron Man", + source_title="Iron Man - 2012 - Remaster", + source_artist="Black Sabbath", + ) + assert ok is True + db.save_sync_match_cache.assert_called_once() + kwargs = db.save_sync_match_cache.call_args.kwargs + assert kwargs["spotify_track_id"] == "spotify-iron-man" + assert kwargs["server_source"] == "plex" + assert kwargs["server_track_id"] == 5001 + assert kwargs["server_track_title"] == "Iron Man" + assert kwargs["confidence"] == 1.0 + assert kwargs["normalized_title"] == "iron man - 2012 - remaster" + assert kwargs["normalized_artist"] == "black sabbath" + + +def test_record_returns_false_when_required_fields_missing(): + db = MagicMock() + assert record_manual_match(db, source_track_id="", server_source="plex", server_track_id=1) is False + assert record_manual_match(db, source_track_id="x", server_source="", server_track_id=1) is False + assert record_manual_match(db, source_track_id="x", server_source="plex", server_track_id=None) is False + db.save_sync_match_cache.assert_not_called() + + +def test_record_returns_false_when_db_save_returns_false(): + db = MagicMock() + db.save_sync_match_cache.return_value = False + assert record_manual_match(db, source_track_id="x", server_source="plex", server_track_id=1) is False + + +def test_record_swallows_db_exception(): + db = MagicMock() + db.save_sync_match_cache.side_effect = RuntimeError("db boom") + assert record_manual_match(db, source_track_id="x", server_source="plex", server_track_id=1) is False + + +def test_record_returns_false_when_db_lacks_method(): + class NoSaveDB: + pass + assert record_manual_match(NoSaveDB(), source_track_id="x", server_source="plex", server_track_id=1) is False + + +def test_record_handles_empty_optional_strings(): + db = MagicMock() + db.save_sync_match_cache.return_value = True + ok = record_manual_match(db, source_track_id="x", server_source="plex", server_track_id=1) + assert ok is True + kwargs = db.save_sync_match_cache.call_args.kwargs + assert kwargs["normalized_title"] == "" + assert kwargs["normalized_artist"] == "" + assert kwargs["server_track_title"] == "" diff --git a/tests/test_acoustid_scanner.py b/tests/test_acoustid_scanner.py index a6d83e12..1993b099 100644 --- a/tests/test_acoustid_scanner.py +++ b/tests/test_acoustid_scanner.py @@ -54,11 +54,11 @@ def _make_context(rows): def test_load_db_tracks_skips_null_ids_and_normalizes_track_ids(): job = AcoustIDScannerJob() context = _make_context([ - # 10 columns: id, title, artist (COALESCE'd), file_path, track_number, + # 11 columns: id, title, artist (COALESCE'd), file_path, track_number, # album_title, album_thumb, artist_thumb, track_artist (raw, may be ''), - # album_artist. - (None, "Broken Track", "Artist", "/music/broken.flac", 1, "Album", None, None, "", "Artist"), - (42, "Good Track", "Artist", "/music/good.flac", 2, "Album", "album-thumb", "artist-thumb", "", "Artist"), + # album_artist, duration_ms (issue #587 — duration guard). + (None, "Broken Track", "Artist", "/music/broken.flac", 1, "Album", None, None, "", "Artist", 180000), + (42, "Good Track", "Artist", "/music/good.flac", 2, "Album", "album-thumb", "artist-thumb", "", "Artist", 240000), ]) tracks = job._load_db_tracks(context) @@ -66,16 +66,17 @@ def test_load_db_tracks_skips_null_ids_and_normalizes_track_ids(): assert list(tracks.keys()) == ["42"] assert tracks["42"]["title"] == "Good Track" assert tracks["42"]["artist"] == "Artist" + assert tracks["42"]["duration_ms"] == 240000 def test_scan_handles_mixed_track_id_types(monkeypatch): job = AcoustIDScannerJob() context = _make_context([ - # 10 columns: id, title, artist (COALESCE'd), file_path, track_number, + # 11 columns: id, title, artist (COALESCE'd), file_path, track_number, # album_title, album_thumb, artist_thumb, track_artist (raw, may be ''), - # album_artist. - (None, "Broken Track", "Artist", "/music/broken.flac", 1, "Album", None, None, "", "Artist"), - (42, "Good Track", "Artist", "/music/good.flac", 2, "Album", "album-thumb", "artist-thumb", "", "Artist"), + # album_artist, duration_ms. + (None, "Broken Track", "Artist", "/music/broken.flac", 1, "Album", None, None, "", "Artist", 180000), + (42, "Good Track", "Artist", "/music/good.flac", 2, "Album", "album-thumb", "artist-thumb", "", "Artist", 240000), ]) monkeypatch.setattr(job, "_resolve_path", lambda file_path, _context: file_path) @@ -275,7 +276,8 @@ def _make_real_db_context(tmp_path): album_id TEXT, file_path TEXT, track_number INTEGER, - track_artist TEXT + track_artist TEXT, + duration INTEGER ); """) conn.commit() @@ -620,3 +622,160 @@ def test_scanner_file_tag_matches_db_no_behavioral_change(monkeypatch): ) assert captured_findings == [] + + +# --------------------------------------------------------------------------- +# Issue #587 — multi-candidate scan + duration guard (Foxxify report) +# --------------------------------------------------------------------------- + + +def test_scanner_no_finding_when_lower_ranked_candidate_matches(): + """Foxxify case 2 — AcoustID returns multiple recordings per + fingerprint; the top match is the wrong-credited recording but a + lower-ranked candidate matches expected metadata exactly. Scanner + should iterate ALL candidates and suppress the finding. + + Repro: file is "Nana" by Geoxor, AcoustID top match is "Nana" by + Edward Vesala Trio (different recording sharing similar + fingerprint), AcoustID's second candidate is the actual Geoxor + track. Pre-fix scanner only saw [0] → flagged. Post-fix sees [1] + → no flag.""" + job = AcoustIDScannerJob() + captured_findings = [] + context = _make_finding_capturing_context( + track_row=("nana", "Nana", "Geoxor", + "/music/nana.opus", 6, "Stardust", None, None), + captured=captured_findings, + ) + + fake_acoustid = SimpleNamespace( + fingerprint_and_lookup=lambda fpath: { + 'best_score': 0.97, + 'recordings': [ + # AcoustID's top match — wrong artist for our file + {'title': 'Nana', 'artist': 'Edward Vesala Trio'}, + # Lower-ranked candidate — actually matches our expected + {'title': 'Nana', 'artist': 'Geoxor'}, + ], + }, + ) + + result = JobResultStub() + job._scan_file( + '/music/nana.opus', 'nana', + {'title': 'Nana', 'artist': 'Geoxor'}, + fake_acoustid, context, result, + fp_threshold=0.85, title_threshold=0.85, artist_threshold=0.6, + ) + + assert captured_findings == [], ( + f"Expected no finding (lower-ranked candidate matches); got {captured_findings}" + ) + + +def test_scanner_still_flags_when_no_candidate_matches(): + """Confirm the multi-candidate check doesn't accidentally suppress + legitimate mismatches — if NO candidate matches expected metadata, + the finding still fires.""" + job = AcoustIDScannerJob() + captured_findings = [] + context = _make_finding_capturing_context( + track_row=("99", "Expected Title", "Expected Artist", + "/music/track.flac", 1, "Album", None, None), + captured=captured_findings, + ) + + fake_acoustid = SimpleNamespace( + fingerprint_and_lookup=lambda fpath: { + 'best_score': 0.99, + 'recordings': [ + {'title': 'Wrong Track', 'artist': 'Wrong Artist A'}, + {'title': 'Different Wrong', 'artist': 'Wrong Artist B'}, + ], + }, + ) + + result = JobResultStub() + job._scan_file( + '/music/track.flac', '99', + {'title': 'Expected Title', 'artist': 'Expected Artist'}, + fake_acoustid, context, result, + fp_threshold=0.85, title_threshold=0.85, artist_threshold=0.6, + ) + + assert len(captured_findings) == 1 + + +def test_scanner_skips_finding_on_strong_duration_mismatch(): + """Foxxify case 3 — 17-minute mashup edit fingerprints to a 5-minute + late-70s Japanese hiphop track. Fingerprint matched a sample/intro + section but the recordings are clearly different (drastic length + difference). Scanner should skip the finding rather than recommend + retag of a totally different track length.""" + job = AcoustIDScannerJob() + captured_findings = [] + context = _make_finding_capturing_context( + track_row=("mashup", "Some Mashup Edit", "Mashup Artist", + "/music/mashup.opus", 1, "Mashups", None, None), + captured=captured_findings, + ) + + # AcoustID matched a 5-minute Japanese hiphop track via fingerprint + # hash collision. Expected file is 17 minutes — duration guard + # should kick in. + fake_acoustid = SimpleNamespace( + fingerprint_and_lookup=lambda fpath: { + 'best_score': 0.98, + 'recordings': [ + {'title': 'Different Song', 'artist': 'Different Artist', + 'duration': 300}, # 5 min — way off from our 17 min file + ], + }, + ) + + result = JobResultStub() + # 17 minutes = 1020 sec = 1020000 ms + job._scan_file( + '/music/mashup.opus', 'mashup', + {'title': 'Some Mashup Edit', 'artist': 'Mashup Artist', 'duration_ms': 1020000}, + fake_acoustid, context, result, + fp_threshold=0.85, title_threshold=0.85, artist_threshold=0.6, + ) + + assert captured_findings == [], ( + f"Expected no finding (duration mismatch suggests collision); got {captured_findings}" + ) + + +def test_scanner_still_flags_when_duration_matches(): + """Confirm the duration guard only kicks in for STRONG mismatches — + similar-length wrong song still gets flagged.""" + job = AcoustIDScannerJob() + captured_findings = [] + context = _make_finding_capturing_context( + track_row=("99", "Expected", "Artist", + "/music/track.flac", 1, "Album", None, None), + captured=captured_findings, + ) + + fake_acoustid = SimpleNamespace( + fingerprint_and_lookup=lambda fpath: { + 'best_score': 0.99, + 'recordings': [ + {'title': 'Wrong Song', 'artist': 'Wrong Artist', + 'duration': 180}, # 3 min, matches expected + ], + }, + ) + + result = JobResultStub() + # 3-minute file with 3-minute candidate — same length, but title + + # artist clearly mismatch → finding should still fire + job._scan_file( + '/music/track.flac', '99', + {'title': 'Expected', 'artist': 'Artist', 'duration_ms': 180000}, + fake_acoustid, context, result, + fp_threshold=0.85, title_threshold=0.85, artist_threshold=0.6, + ) + + assert len(captured_findings) == 1 diff --git a/tests/test_album_bundle.py b/tests/test_album_bundle.py new file mode 100644 index 00000000..48733c31 --- /dev/null +++ b/tests/test_album_bundle.py @@ -0,0 +1,301 @@ +"""Tests for ``core/download_plugins/album_bundle.py``. + +The shared helpers used by both the torrent and usenet album-bundle +flows. Pins the pick heuristic, the atomic-copy invariant +(no partial files ever visible at the audio extension), the +collision-suffix logic, and the config-driven poll cadence so a +future tweak in either plugin can't break the contract. +""" + +from __future__ import annotations + +import os +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional +from unittest.mock import patch + +import pytest + +from core.download_plugins.album_bundle import ( + ALBUM_PICK_MAX_BYTES, + ALBUM_PICK_MIN_BYTES, + DEFAULT_POLL_INTERVAL_SECONDS, + DEFAULT_POLL_TIMEOUT_SECONDS, + atomic_copy_to_staging, + copy_audio_files_atomically, + get_poll_interval, + get_poll_timeout, + pick_best_album_release, + quality_score, + unique_staging_path, +) + + +# Minimal release-result shim — duck-types the fields the picker reads. +@dataclass +class _Release: + title: str + size: int + seeders: Optional[int] = None + grabs: Optional[int] = None + + +def _flac_quality_guess(title: str) -> str: + """Stand-in for the plugin's title→quality function.""" + t = (title or '').lower() + if 'flac' in t: + return 'flac' + if 'aac' in t: + return 'aac' + if 'ogg' in t: + return 'ogg' + return 'mp3' + + +# --------------------------------------------------------------------------- +# pick_best_album_release +# --------------------------------------------------------------------------- + + +def test_picker_returns_none_for_empty_input() -> None: + assert pick_best_album_release([], _flac_quality_guess) is None + + +def test_picker_drops_singletons_when_albums_present() -> None: + """Single-track torrents under 40 MB shouldn't beat an album-sized + candidate even if the single has thousands of seeders.""" + single = _Release(title='Track [MP3]', size=10_000_000, seeders=10_000) + album = _Release(title='Album [MP3]', size=120_000_000, seeders=5) + assert pick_best_album_release([single, album], _flac_quality_guess) is album + + +def test_picker_prefers_flac_when_tied_on_seeders() -> None: + flac = _Release(title='Album [FLAC]', size=400_000_000, seeders=50) + mp3 = _Release(title='Album [MP3]', size=130_000_000, seeders=50) + assert pick_best_album_release([flac, mp3], _flac_quality_guess) is flac + + +def test_picker_uses_grabs_when_seeders_is_none() -> None: + """Usenet results have ``seeders=None`` — the picker should fall + back to ``grabs`` so popularity still drives the ranking.""" + cold = _Release(title='Album A [MP3]', size=200_000_000, seeders=None, grabs=1) + popular = _Release(title='Album B [MP3]', size=200_000_000, seeders=None, grabs=999) + assert pick_best_album_release([cold, popular], _flac_quality_guess) is popular + + +def test_picker_falls_back_when_all_below_floor() -> None: + """When every candidate is below the 40 MB album-size floor, + return the most-seeded one rather than None — the user still + wants a download attempt.""" + small_low = _Release(title='X', size=5_000_000, seeders=10) + small_high = _Release(title='Y', size=8_000_000, seeders=200) + assert pick_best_album_release([small_low, small_high], _flac_quality_guess) is small_high + + +def test_picker_size_floor_matches_constant() -> None: + """If someone moves the constant the floor moves with it — pin + the relationship to catch accidental literals creeping back in.""" + just_below = _Release(title='Below', size=ALBUM_PICK_MIN_BYTES - 1, seeders=999) + just_above = _Release(title='Above', size=ALBUM_PICK_MIN_BYTES + 1, seeders=1) + assert pick_best_album_release([just_below, just_above], _flac_quality_guess) is just_above + + +def test_picker_rejects_oversized_box_sets() -> None: + """Anything past 3 GB drops out of the preferred pool — most likely + a multi-disc box set with scans + bonus material, not what the + user asked for.""" + sane = _Release(title='Album [FLAC]', size=400_000_000, seeders=10) + box = _Release(title='Album Box [FLAC]', size=ALBUM_PICK_MAX_BYTES + 1_000_000, seeders=999) + # Sane wins even with 100x fewer seeders, because box is outside + # the preferred range. + assert pick_best_album_release([sane, box], _flac_quality_guess) is sane + + +# --------------------------------------------------------------------------- +# quality_score +# --------------------------------------------------------------------------- + + +def test_quality_score_orders_formats() -> None: + assert quality_score('Album [FLAC]', _flac_quality_guess) > quality_score('Album [MP3]', _flac_quality_guess) + assert quality_score('Album [AAC]', _flac_quality_guess) > quality_score('Album [MP3]', _flac_quality_guess) + assert quality_score('Bare title', _flac_quality_guess) == quality_score('Album [MP3]', _flac_quality_guess) + + +# --------------------------------------------------------------------------- +# unique_staging_path +# --------------------------------------------------------------------------- + + +def test_unique_staging_path_returns_natural_when_clear(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + assert unique_staging_path(staging, src) == staging / 'src.flac' + + +def test_unique_staging_path_suffixes_on_collision(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + (staging / 'src.flac').write_bytes(b'existing') + assert unique_staging_path(staging, src) == staging / 'src_1.flac' + + +def test_unique_staging_path_increments_suffix(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC') + staging = tmp_path / 'staging' + staging.mkdir() + (staging / 'src.flac').write_bytes(b'1') + (staging / 'src_1.flac').write_bytes(b'2') + (staging / 'src_2.flac').write_bytes(b'3') + assert unique_staging_path(staging, src) == staging / 'src_3.flac' + + +# --------------------------------------------------------------------------- +# atomic_copy_to_staging +# --------------------------------------------------------------------------- + + +def test_atomic_copy_lands_at_final_path(tmp_path: Path) -> None: + src = tmp_path / 'src.flac' + src.write_bytes(b'fLaC payload') + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + assert atomic_copy_to_staging(src, dest) is True + assert dest.read_bytes() == b'fLaC payload' + + +def test_atomic_copy_leaves_no_tmp_files_after_success(tmp_path: Path) -> None: + """The .tmp. sidecar must be cleaned up by the rename — + no orphan files left behind on a successful copy.""" + src = tmp_path / 'src.flac' + src.write_bytes(b'data') + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + atomic_copy_to_staging(src, dest) + tmp_files = list(dest.parent.glob('*.tmp.*')) + assert tmp_files == [] + + +def test_atomic_copy_never_exposes_partial_to_extension_scanner(tmp_path: Path) -> None: + """Auto-Import filters by audio extension — the in-flight file + must NEVER be visible at its final extension until the copy is + complete. We probe this by scanning the staging dir in parallel + with the copy and assert the audio file is either absent OR + fully written. + """ + src = tmp_path / 'src.flac' + src.write_bytes(b'x' * (2 * 1024 * 1024)) + dest = tmp_path / 'staging' / 'track.flac' + dest.parent.mkdir() + + stop = threading.Event() + saw_partial = threading.Event() + expected_size = src.stat().st_size + + def _scan_loop(): + while not stop.is_set(): + try: + files = [p for p in dest.parent.iterdir() if p.suffix == '.flac'] + except FileNotFoundError: + continue + for fp in files: + size = fp.stat().st_size + if 0 < size < expected_size: + saw_partial.set() + return + + scanner = threading.Thread(target=_scan_loop, daemon=True) + scanner.start() + try: + for i in range(5): + target = dest.with_name(f'track_{i}.flac') + atomic_copy_to_staging(src, target) + # Give the scanner a moment to drain any final scan iteration. + time.sleep(0.05) + finally: + stop.set() + scanner.join(timeout=1.0) + + assert not saw_partial.is_set(), \ + "Scanner observed a partial audio file — atomic copy contract broken" + + +def test_copy_audio_files_atomically_skips_failures(tmp_path: Path) -> None: + """One file failing to copy shouldn't stop the rest from being + staged — partial results are better than a complete bailout.""" + src_a = tmp_path / 'a.flac' + src_a.write_bytes(b'a') + src_missing = tmp_path / 'does-not-exist.flac' # never created + src_c = tmp_path / 'c.flac' + src_c.write_bytes(b'c') + staging = tmp_path / 'staging' + out = copy_audio_files_atomically([src_a, src_missing, src_c], staging) + assert len(out) == 2 + landed = sorted(Path(p).name for p in out) + assert landed == ['a.flac', 'c.flac'] + + +def test_copy_audio_files_atomically_creates_staging_dir(tmp_path: Path) -> None: + src = tmp_path / 'a.flac' + src.write_bytes(b'a') + staging = tmp_path / 'nested' / 'staging' / 'dir' + out = copy_audio_files_atomically([src], staging) + assert len(out) == 1 + assert staging.exists() + + +# --------------------------------------------------------------------------- +# Config-driven poll cadence +# --------------------------------------------------------------------------- + + +def test_get_poll_interval_uses_default_when_unset() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = DEFAULT_POLL_INTERVAL_SECONDS + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + + +def test_get_poll_interval_honours_override() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 5 + assert get_poll_interval() == 5.0 + + +def test_get_poll_interval_falls_back_on_garbage() -> None: + """Non-numeric / non-positive values fall back to the default + rather than crashing the poll loop.""" + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 'not-a-number' + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + cm.get.return_value = -1 + assert get_poll_interval() == DEFAULT_POLL_INTERVAL_SECONDS + + +def test_get_poll_timeout_uses_default_when_unset() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = DEFAULT_POLL_TIMEOUT_SECONDS + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS + + +def test_get_poll_timeout_honours_override() -> None: + """Users with slow trackers / large box sets can extend the + deadline without touching code.""" + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 86_400 # 24h + assert get_poll_timeout() == 86_400.0 + + +def test_get_poll_timeout_falls_back_on_garbage() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = '' + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS + cm.get.return_value = 0 + assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS diff --git a/tests/test_album_bundle_dispatch.py b/tests/test_album_bundle_dispatch.py new file mode 100644 index 00000000..5cafa065 --- /dev/null +++ b/tests/test_album_bundle_dispatch.py @@ -0,0 +1,365 @@ +"""Tests for ``core/downloads/album_bundle_dispatch.py``. + +Pins the gate predicate, the resolution + run flow, and the +fail / fall-through return contract. Mocks the config, plugin +resolver, and state access so the dispatcher is testable without +standing up runtime_state or a real plugin. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from core.downloads.album_bundle_dispatch import ( + BatchStateAccess, + is_eligible, + try_dispatch, +) + + +class _FakeState: + """In-memory ``BatchStateAccess`` for tests — records every + update so assertions can check the sequence of fields set.""" + + def __init__(self) -> None: + self.fields: dict = {} + self.update_calls: list = [] + self.failed_with: str = '' + + def update_fields(self, batch_id: str, fields: dict) -> None: + self.update_calls.append((batch_id, dict(fields))) + self.fields.update(fields) + + def mark_failed(self, batch_id: str, error: str) -> None: + self.failed_with = error + self.fields['phase'] = 'failed' + self.fields['error'] = error + self.fields['album_bundle_state'] = 'failed' + + +def _config(values: dict): + """Build a config_get callable from a flat dict.""" + def _get(key, default=None): + return values.get(key, default) + return _get + + +# --------------------------------------------------------------------------- +# is_eligible pure predicate +# --------------------------------------------------------------------------- + + +def test_is_eligible_requires_album_flag() -> None: + assert is_eligible(mode='torrent', is_album=False, + album_name='X', artist_name='Y') is False + + +def test_is_eligible_requires_album_bundle_mode() -> None: + for mode in ('youtube', 'tidal', 'qobuz', 'hifi', + 'deezer_dl', 'amazon', 'lidarr', 'soundcloud', 'hybrid'): + assert is_eligible(mode=mode, is_album=True, + album_name='X', artist_name='Y') is False + + +def test_is_eligible_accepts_torrent_usenet_and_soulseek() -> None: + assert is_eligible(mode='torrent', is_album=True, + album_name='X', artist_name='Y') is True + assert is_eligible(mode='usenet', is_album=True, + album_name='X', artist_name='Y') is True + assert is_eligible(mode='soulseek', is_album=True, + album_name='X', artist_name='Y') is True + + +def test_is_eligible_requires_non_empty_names() -> None: + assert is_eligible(mode='torrent', is_album=True, + album_name='', artist_name='Y') is False + assert is_eligible(mode='torrent', is_album=True, + album_name='X', artist_name='') is False + assert is_eligible(mode='torrent', is_album=True, + album_name=' ', artist_name='Y') is False + + +def test_is_eligible_case_insensitive_mode() -> None: + assert is_eligible(mode='TORRENT', is_album=True, + album_name='X', artist_name='Y') is True + + +# --------------------------------------------------------------------------- +# try_dispatch — gate evaluation +# --------------------------------------------------------------------------- + + +def test_dispatch_returns_false_when_not_album() -> None: + state = _FakeState() + plugin = MagicMock() + result = try_dispatch( + batch_id='b1', is_album=False, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + assert state.update_calls == [] + plugin.download_album_to_staging.assert_not_called() + + +def test_dispatch_returns_false_for_non_album_bundle_modes() -> None: + state = _FakeState() + plugin = MagicMock() + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'youtube'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_plugin_missing() -> None: + """No plugin available → fall through to per-track flow with a + warning. The state SHOULD NOT have been touched.""" + state = _FakeState() + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: None, state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_plugin_lacks_method() -> None: + state = _FakeState() + # Plugin that doesn't implement download_album_to_staging. + class _LegacyPlugin: + pass + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: _LegacyPlugin(), state=state, + ) + assert result is False + assert state.update_calls == [] + + +def test_dispatch_returns_false_when_resolver_raises() -> None: + """Plugin resolution can fail (registry not initialised); we log + and fall through rather than crashing the master worker.""" + state = _FakeState() + def _boom(_name): + raise RuntimeError("registry not initialised") + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'X'}, artist_context={'name': 'Y'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=_boom, state=state, + ) + assert result is False + + +# --------------------------------------------------------------------------- +# try_dispatch — success / failure paths +# --------------------------------------------------------------------------- + + +def test_dispatch_success_returns_false_so_per_track_can_run() -> None: + """Success → master worker should CONTINUE to per-track flow so + each task can hit try_staging_match and find its file.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = { + 'success': True, 'files': ['/tmp/a.flac', '/tmp/b.flac'], 'error': None, + } + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({ + 'download_source.mode': 'torrent', + 'import.staging_path': '/staging/path', + }), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + # Plugin was called with the right args. + args = plugin.download_album_to_staging.call_args + assert args.args[0] == 'GNX' + assert args.args[1] == 'Kendrick Lamar' + assert args.args[2].replace('\\', '/').endswith('storage/album_bundle_staging/b1') + # Phase transitioned through searching → analysis. + assert state.fields['phase'] == 'analysis' + assert state.fields['album_bundle_state'] == 'staged' + assert state.fields['album_bundle_source'] == 'torrent' + assert state.fields['album_bundle_private_staging'] is True + assert state.fields['album_bundle_staging_path'].replace('\\', '/').endswith('storage/album_bundle_staging/b1') + assert state.failed_with == '' + + +def test_dispatch_uses_configured_private_album_bundle_staging_root() -> None: + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/tmp/a.flac']} + + try_dispatch( + batch_id='batch:with/slash', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({ + 'download_source.mode': 'torrent', + 'download_source.album_bundle_staging_path': '/private/staging', + }), + plugin_resolver=lambda _name: plugin, state=state, + ) + + staging_arg = plugin.download_album_to_staging.call_args.args[2].replace('\\', '/') + assert staging_arg == '/private/staging/batch_with_slash' + assert state.fields['album_bundle_staging_path'].replace('\\', '/') == staging_arg + + +def test_dispatch_failure_returns_true_so_master_stops() -> None: + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = { + 'success': False, 'files': [], 'error': 'No torrent results found', + } + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is True + assert state.failed_with == 'No torrent results found' + assert state.fields['phase'] == 'failed' + + +def test_dispatch_fallback_failure_returns_false_for_per_track_flow() -> None: + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = { + 'success': False, + 'files': [], + 'error': 'No complete Soulseek album folders found', + 'fallback': True, + } + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'Album'}, artist_context={'name': 'Artist'}, + config_get=_config({'download_source.mode': 'soulseek'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is False + assert state.failed_with == '' + assert state.fields['phase'] == 'analysis' + assert state.fields['album_bundle_state'] == 'fallback' + assert state.fields['album_bundle_error'] == 'No complete Soulseek album folders found' + + +def test_dispatch_plugin_exception_treated_as_failure() -> None: + """A bug / network error in the plugin must not propagate into + the master worker — caught + treated as a normal failure so + the batch reports the error cleanly.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.side_effect = RuntimeError("network down") + result = try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + assert result is True + assert 'network down' in state.failed_with + + +def test_dispatch_strips_whitespace_from_names() -> None: + """Trailing whitespace in batch context shouldn't fail the + eligibility predicate AND should be cleaned before passing to + the plugin.""" + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/x']} + try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': ' GNX '}, artist_context={'name': ' Kendrick '}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + args = plugin.download_album_to_staging.call_args + assert args.args[0] == 'GNX' + assert args.args[1] == 'Kendrick' + + +def test_dispatch_source_override_uses_first_hybrid_source() -> None: + state = _FakeState() + plugin = MagicMock() + plugin.download_album_to_staging.return_value = {'success': True, 'files': ['/x']} + seen = [] + + try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'hybrid'}), + plugin_resolver=lambda name: seen.append(name) or plugin, + state=state, + source_override='soulseek', + ) + + assert seen == ['soulseek'] + assert state.fields['album_bundle_source'] == 'soulseek' + + +def test_dispatch_progress_callback_mirrors_payload_to_state() -> None: + """The progress callback the plugin gets must mirror its + payload onto the batch state under ``album_bundle_*`` keys so + the Downloads page can render progress while the torrent + download runs.""" + state = _FakeState() + captured_emit = {} + + def _capture(album, artist, staging, emit): + captured_emit['fn'] = emit + emit({'state': 'searching', 'release': 'GNX [FLAC]'}) + emit({'state': 'downloading', 'progress': 0.42, 'speed': 1024 * 1024}) + emit({'state': 'staged', 'count': 12}) + return {'success': True, 'files': []} + + plugin = MagicMock() + plugin.download_album_to_staging.side_effect = _capture + try_dispatch( + batch_id='b1', is_album=True, + album_context={'name': 'GNX'}, artist_context={'name': 'Kendrick Lamar'}, + config_get=_config({'download_source.mode': 'torrent'}), + plugin_resolver=lambda _name: plugin, state=state, + ) + # State should have seen each of the three lifecycle emissions. + states_seen = [fields.get('album_bundle_state') + for _, fields in state.update_calls + if 'album_bundle_state' in fields] + assert 'searching' in states_seen + assert 'downloading' in states_seen + assert 'staged' in states_seen + # Numeric progress + release name made it through. + assert state.fields['album_bundle_release'] == 'GNX [FLAC]' + assert state.fields['album_bundle_progress'] == 0.42 + assert state.fields['album_bundle_count'] == 12 + + +# --------------------------------------------------------------------------- +# Protocol conformance — runtime impl must satisfy the contract +# --------------------------------------------------------------------------- + + +def test_runtime_state_impl_matches_protocol() -> None: + """Sanity check that the concrete BatchStateAccess impl in + master.py implements both methods. We don't import master.py + here (would pull in heavy deps); duck-check on the _FakeState + instead since it's a sibling impl of the same Protocol.""" + state: BatchStateAccess = _FakeState() + state.update_fields('b1', {'x': 1}) + state.mark_failed('b1', 'oops') + assert state.fields['x'] == 1 + assert state.fields['error'] == 'oops' diff --git a/tests/test_archive_pipeline.py b/tests/test_archive_pipeline.py new file mode 100644 index 00000000..053c200a --- /dev/null +++ b/tests/test_archive_pipeline.py @@ -0,0 +1,238 @@ +"""Tests for ``core/archive_pipeline.py``. + +Covers the audio-file walker, archive detector, and zip / tar +extraction (rar / 7z paths use optional deps so they're only +exercised when the libs are present in the test environment). +Path-traversal protection gets explicit coverage — a malicious +archive must not escape the extraction directory. +""" + +from __future__ import annotations + +import os +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from core.archive_pipeline import ( + AUDIO_EXTENSIONS, + ARCHIVE_EXTENSIONS, + collect_audio_after_extraction, + extract_archive, + extract_all_in_dir, + find_archives_in_dir, + is_archive, + walk_audio_files, +) + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + + +def test_audio_extensions_cover_common_formats() -> None: + for ext in ('.flac', '.mp3', '.m4a', '.ogg', '.opus', '.wav', '.aac', '.wma'): + assert ext in AUDIO_EXTENSIONS + + +def test_archive_extensions_cover_common_formats() -> None: + for ext in ('.zip', '.rar', '.tar', '.7z'): + assert ext in ARCHIVE_EXTENSIONS + + +# --------------------------------------------------------------------------- +# is_archive +# --------------------------------------------------------------------------- + + +def test_is_archive_detects_simple_extensions(tmp_path: Path) -> None: + zip_path = tmp_path / 'x.zip' + zip_path.write_bytes(b'PK\x03\x04') # minimal — is_archive doesn't validate content + assert is_archive(zip_path) is True + + +def test_is_archive_detects_compound_tar_extensions(tmp_path: Path) -> None: + """``.tar.gz`` etc. — Path.suffix only catches the last suffix, + so the detector has to special-case compound extensions.""" + targz = tmp_path / 'x.tar.gz' + targz.write_bytes(b'\x1f\x8b') + assert is_archive(targz) is True + + +def test_is_archive_returns_false_for_audio(tmp_path: Path) -> None: + flac = tmp_path / 'song.flac' + flac.write_bytes(b'fLaC') + assert is_archive(flac) is False + + +def test_is_archive_returns_false_for_missing_file(tmp_path: Path) -> None: + assert is_archive(tmp_path / 'does-not-exist.zip') is False + + +# --------------------------------------------------------------------------- +# walk_audio_files +# --------------------------------------------------------------------------- + + +def test_walk_audio_files_finds_nested(tmp_path: Path) -> None: + # Layout: root/album/disc1/track.flac + root/album/disc2/track.mp3 + (tmp_path / 'album' / 'disc1').mkdir(parents=True) + (tmp_path / 'album' / 'disc2').mkdir(parents=True) + (tmp_path / 'album' / 'disc1' / 'track1.flac').write_bytes(b'fLaC') + (tmp_path / 'album' / 'disc2' / 'track1.mp3').write_bytes(b'ID3') + (tmp_path / 'album' / 'cover.jpg').write_bytes(b'\xff\xd8') + found = walk_audio_files(tmp_path) + names = sorted(p.name for p in found) + assert names == ['track1.flac', 'track1.mp3'] + + +def test_walk_audio_files_returns_empty_for_missing(tmp_path: Path) -> None: + assert walk_audio_files(tmp_path / 'does-not-exist') == [] + + +def test_walk_audio_files_ignores_non_audio(tmp_path: Path) -> None: + (tmp_path / 'readme.txt').write_text('hi') + (tmp_path / 'cover.png').write_bytes(b'\x89PNG') + assert walk_audio_files(tmp_path) == [] + + +def test_walk_audio_files_case_insensitive_extension(tmp_path: Path) -> None: + """Lots of torrents have uppercase extensions (.MP3, .FLAC) — + the walker must catch those too.""" + (tmp_path / 'TRACK.MP3').write_bytes(b'ID3') + (tmp_path / 'TRACK.FLAC').write_bytes(b'fLaC') + found = walk_audio_files(tmp_path) + assert len(found) == 2 + + +# --------------------------------------------------------------------------- +# find_archives_in_dir +# --------------------------------------------------------------------------- + + +def test_find_archives_in_dir_only_top_level(tmp_path: Path) -> None: + """find_archives doesn't recurse — torrents put the archive at + the top of the dir; deeper search risks extracting unrelated + archives that ship inside a sample folder, etc.""" + (tmp_path / 'album.zip').write_bytes(b'PK\x03\x04') + nested = tmp_path / 'subdir' + nested.mkdir() + (nested / 'nested.zip').write_bytes(b'PK\x03\x04') + found = find_archives_in_dir(tmp_path) + assert [p.name for p in found] == ['album.zip'] + + +def test_find_archives_in_dir_empty(tmp_path: Path) -> None: + assert find_archives_in_dir(tmp_path) == [] + assert find_archives_in_dir(tmp_path / 'missing') == [] + + +# --------------------------------------------------------------------------- +# extract_archive — zip +# --------------------------------------------------------------------------- + + +def test_extract_zip_writes_files(tmp_path: Path) -> None: + zip_path = tmp_path / 'album.zip' + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('track1.mp3', b'ID3 track1 data') + zf.writestr('track2.flac', b'fLaC data') + result = extract_archive(zip_path) + assert result == zip_path.parent + assert (tmp_path / 'track1.mp3').exists() + assert (tmp_path / 'track2.flac').exists() + + +def test_extract_zip_rejects_path_traversal(tmp_path: Path) -> None: + """A malicious archive trying to write ``../../etc/passwd`` must + be refused without extracting anything.""" + zip_path = tmp_path / 'evil.zip' + extract_dest = tmp_path / 'staging' + extract_dest.mkdir() + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('../escaped.txt', b'evil') + zf.writestr('safe.mp3', b'ID3') + extract_archive(zip_path, extract_to=extract_dest) + # Neither file should have landed — extraction aborts on the first + # traversal attempt. + assert not (extract_dest / 'safe.mp3').exists() + assert not (tmp_path / 'escaped.txt').exists() + + +def test_extract_zip_returns_none_for_bad_zip(tmp_path: Path) -> None: + bad = tmp_path / 'not-a-zip.zip' + bad.write_bytes(b'this is not a zip') + assert extract_archive(bad) is None + + +def test_extract_archive_missing_file_returns_none(tmp_path: Path) -> None: + assert extract_archive(tmp_path / 'does-not-exist.zip') is None + + +# --------------------------------------------------------------------------- +# extract_archive — tar +# --------------------------------------------------------------------------- + + +def test_extract_tar_gz_writes_files(tmp_path: Path) -> None: + payload = tmp_path / 'track.mp3' + payload.write_bytes(b'ID3 track') + tar_path = tmp_path / 'album.tar.gz' + with tarfile.open(tar_path, 'w:gz') as tf: + tf.add(payload, arcname='track.mp3') + payload.unlink() # remove the source so we can verify the extract recreated it + extract_archive(tar_path) + assert (tmp_path / 'track.mp3').exists() + + +def test_extract_tar_rejects_path_traversal(tmp_path: Path) -> None: + extract_dest = tmp_path / 'staging' + extract_dest.mkdir() + tar_path = tmp_path / 'evil.tar' + payload = tmp_path / 'src.txt' + payload.write_bytes(b'evil') + with tarfile.open(tar_path, 'w') as tf: + info = tf.gettarinfo(str(payload), arcname='../escaped.txt') + with payload.open('rb') as fh: + tf.addfile(info, fh) + extract_archive(tar_path, extract_to=extract_dest) + assert not (tmp_path / 'escaped.txt').exists() + + +# --------------------------------------------------------------------------- +# extract_all_in_dir + collect_audio_after_extraction +# --------------------------------------------------------------------------- + + +def test_extract_all_in_dir_handles_multiple_archives(tmp_path: Path) -> None: + (tmp_path / 'one.zip') # placeholder + z1 = tmp_path / 'one.zip' + z2 = tmp_path / 'two.zip' + with zipfile.ZipFile(z1, 'w') as zf: + zf.writestr('a.mp3', b'a') + with zipfile.ZipFile(z2, 'w') as zf: + zf.writestr('b.mp3', b'b') + out = extract_all_in_dir(tmp_path) + assert len(out) == 2 + assert (tmp_path / 'a.mp3').exists() + assert (tmp_path / 'b.mp3').exists() + + +def test_collect_audio_after_extraction_combines_loose_and_extracted(tmp_path: Path) -> None: + """The typical mixed case: torrent dropped a .zip and also some + loose .mp3 files alongside it. The collector returns BOTH.""" + (tmp_path / 'bonus.mp3').write_bytes(b'ID3') + zip_path = tmp_path / 'main.zip' + with zipfile.ZipFile(zip_path, 'w') as zf: + zf.writestr('track1.flac', b'fLaC') + zf.writestr('track2.flac', b'fLaC') + found = collect_audio_after_extraction(tmp_path) + names = sorted(p.name for p in found) + assert names == ['bonus.mp3', 'track1.flac', 'track2.flac'] + + +def test_collect_audio_after_extraction_no_archives_no_audio(tmp_path: Path) -> None: + assert collect_audio_after_extraction(tmp_path) == [] diff --git a/tests/test_database_io_resilience.py b/tests/test_database_io_resilience.py new file mode 100644 index 00000000..37da1cc7 --- /dev/null +++ b/tests/test_database_io_resilience.py @@ -0,0 +1,98 @@ +import sqlite3 + +from database.music_database import MusicDatabase + + +def test_clear_server_data_does_not_fail_when_vacuum_hits_disk_io(): + db = object.__new__(MusicDatabase) + + class _Cursor: + rowcount = 0 + + def __init__(self): + self.calls = [] + + def execute(self, query, params=None): + self.calls.append((query, params)) + if query == "VACUUM": + raise sqlite3.OperationalError("disk I/O error") + if "tracks" in query: + self.rowcount = 1500 + elif "albums" in query: + self.rowcount = 200 + elif "artists" in query: + self.rowcount = 20 + + class _Conn: + def __init__(self): + self.cursor_obj = _Cursor() + self.commits = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_obj + + def commit(self): + self.commits += 1 + + conn = _Conn() + db._get_connection = lambda: conn + + db.clear_server_data("jellyfin") + + assert conn.commits == 1 + assert any(call[0] == "VACUUM" for call in conn.cursor_obj.calls) + + +def test_clear_server_data_retries_transient_disk_io_before_commit(monkeypatch): + db = object.__new__(MusicDatabase) + connections = [] + + class _Cursor: + rowcount = 0 + + def __init__(self, fail_first_delete=False): + self.fail_first_delete = fail_first_delete + self.calls = [] + + def execute(self, query, params=None): + self.calls.append((query, params)) + if self.fail_first_delete and "DELETE FROM tracks" in query: + self.fail_first_delete = False + raise sqlite3.OperationalError("disk I/O error") + self.rowcount = 1 + + class _Conn: + def __init__(self, fail_first_delete=False): + self.cursor_obj = _Cursor(fail_first_delete=fail_first_delete) + self.commits = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_obj + + def commit(self): + self.commits += 1 + + def _connect(): + conn = _Conn(fail_first_delete=not connections) + connections.append(conn) + return conn + + db._get_connection = _connect + monkeypatch.setattr("database.music_database.time.sleep", lambda _seconds: None) + + db.clear_server_data("jellyfin") + + assert len(connections) == 2 + assert connections[1].commits == 1 diff --git a/tests/test_download_plugin_conformance.py b/tests/test_download_plugin_conformance.py index 2b0f4e9d..68483adc 100644 --- a/tests/test_download_plugin_conformance.py +++ b/tests/test_download_plugin_conformance.py @@ -59,6 +59,7 @@ def _import_plugin_classes(): from core.deezer_download_client import DeezerDownloadClient from core.lidarr_download_client import LidarrDownloadClient from core.soundcloud_client import SoundcloudClient + from core.amazon_download_client import AmazonDownloadClient return { 'soulseek': SoulseekClient, @@ -69,10 +70,11 @@ def _import_plugin_classes(): 'deezer': DeezerDownloadClient, 'lidarr': LidarrDownloadClient, 'soundcloud': SoundcloudClient, + 'amazon': AmazonDownloadClient, } -def test_default_registry_registers_all_eight_sources(): +def test_default_registry_registers_all_sources(): """Smoke check that the foundation registry knows about every source the orchestrator historically dispatched to. If someone drops a registration here, every other test in this module would @@ -82,7 +84,8 @@ def test_default_registry_registers_all_eight_sources(): registry = build_default_registry() expected = { 'soulseek', 'youtube', 'tidal', 'qobuz', - 'hifi', 'deezer', 'lidarr', 'soundcloud', + 'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon', + 'torrent', 'usenet', } assert set(registry.names()) == expected @@ -102,7 +105,7 @@ def test_deezer_dl_alias_is_registered_against_deezer_spec(): @pytest.mark.parametrize('plugin_name', [ 'soulseek', 'youtube', 'tidal', 'qobuz', - 'hifi', 'deezer', 'lidarr', 'soundcloud', + 'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon', ]) def test_plugin_class_has_all_required_methods(plugin_name): """Every registered plugin class exposes every protocol method @@ -122,7 +125,7 @@ def test_plugin_class_has_all_required_methods(plugin_name): @pytest.mark.parametrize('plugin_name', [ 'soulseek', 'youtube', 'tidal', 'qobuz', - 'hifi', 'deezer', 'lidarr', 'soundcloud', + 'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon', ]) def test_plugin_class_async_methods_are_coroutines(plugin_name): """Methods declared async in the protocol must be async on every diff --git a/tests/test_image_cache.py b/tests/test_image_cache.py new file mode 100644 index 00000000..f4556757 --- /dev/null +++ b/tests/test_image_cache.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from core.image_cache import ImageCache + + +class FakeResponse: + def __init__(self, body: bytes, *, status_code: int = 200, content_type: str = "image/jpeg"): + self.body = body + self.status_code = status_code + self.headers = { + "Content-Type": content_type, + "Content-Length": str(len(body)), + } + self.closed = False + + def iter_content(self, chunk_size=65536): + yield self.body + + def close(self): + self.closed = True + + +def test_cache_url_for_registers_hashed_browser_path(tmp_path): + cache = ImageCache(tmp_path) + url = "https://images.example.test/cover.jpg?token=secret" + + cached_url = cache.cache_url_for(url) + + assert cached_url == f"/api/image-cache/{ImageCache.key_for_url(url)}" + assert "secret" not in cached_url + + +def test_get_url_fetches_once_then_serves_cached_file(tmp_path): + calls = [] + + def fetcher(url, **kwargs): + calls.append((url, kwargs)) + return FakeResponse(b"fake-jpeg-bytes") + + cache = ImageCache(tmp_path, fetcher=fetcher) + url = "https://images.example.test/cover.jpg" + + first = cache.get_url(url) + second = cache.get_url(url) + + assert first.status == "miss" + assert second.status == "hit" + assert first.path == second.path + assert first.path.read_bytes() == b"fake-jpeg-bytes" + assert first.mime_type == "image/jpeg" + assert len(calls) == 1 + + +def test_get_url_rejects_non_image_responses(tmp_path): + cache = ImageCache( + tmp_path, + fetcher=lambda url, **kwargs: FakeResponse(b"", content_type="text/html"), + ) + + try: + cache.get_url("https://images.example.test/not-image") + except Exception as exc: + assert "not an image" in str(exc) + else: + raise AssertionError("Expected non-image response to fail") + diff --git a/tests/test_import_page_album_lookup_pattern.py b/tests/test_import_page_album_lookup_pattern.py index 54bf142d..5bf4ab9d 100644 --- a/tests/test_import_page_album_lookup_pattern.py +++ b/tests/test_import_page_album_lookup_pattern.py @@ -82,19 +82,39 @@ def test_select_album_handler_reads_cache(js_source: str): ) -def test_card_renderers_populate_cache_before_onclick(js_source: str): - """Both renderers (suggestion card + search-result card) must write - to ``_albumLookup`` before emitting the onclick — otherwise the - click handler reads an empty cache for newly-displayed albums.""" - cache_writes = re.findall( - r"_albumLookup\[a\.id\]\s*=\s*\{", - js_source, +def test_card_renderer_populates_cache_before_onclick(js_source: str): + """The shared card-renderer ``_renderSuggestionCard`` must write to + ``_albumLookup`` before emitting the onclick — otherwise the click + handler reads an empty cache for newly-displayed albums. + + Originally this test required >=2 cache writes (one per inline + renderer), but the search-results inline render was consolidated + into a single ``_renderSuggestionCard`` call as part of the #681 + fix. The invariant now is: the shared renderer populates the cache, + and every render call site goes through it (no inline duplicates).""" + # 1. The shared renderer must contain the cache write. + match = re.search( + r"function _renderSuggestionCard\([^)]*\) \{(.*?)^\}", + js_source, re.DOTALL | re.MULTILINE, ) - assert len(cache_writes) >= 2, ( - f"Expected >=2 _albumLookup writes (one per card renderer - " - f"suggestions + search results), found {len(cache_writes)}. " - "Adding a new card-rendering site without populating the cache " - "regresses issue #524 for that path." + assert match, "_renderSuggestionCard function not found" + body = match.group(1) + assert re.search(r"_albumLookup\[a\.id\]\s*=\s*\{", body), ( + "_renderSuggestionCard no longer writes to _albumLookup before " + "emitting the onclick — every card rendered through this helper " + "would have an empty cache on click, regressing issue #524." + ) + + # 2. No inline card render allowed outside the shared helper. + # A second `_albumLookup[a.id] = {` write means a caller is + # re-implementing the renderer instead of calling the helper — + # that's exactly the duplication the #524 fix consolidated away. + cache_writes = re.findall(r"_albumLookup\[a\.id\]\s*=\s*\{", js_source) + assert len(cache_writes) == 1, ( + f"Expected exactly 1 _albumLookup write (inside _renderSuggestionCard), " + f"found {len(cache_writes)}. A new inline card-render site has " + "duplicated the cache-write logic — route the new caller through " + "_renderSuggestionCard(a, primarySource) instead." ) diff --git a/tests/test_library_disk_usage.py b/tests/test_library_disk_usage.py index 00707aa7..1214da8c 100644 --- a/tests/test_library_disk_usage.py +++ b/tests/test_library_disk_usage.py @@ -69,6 +69,53 @@ def test_file_size_column_exists_after_init(db: MusicDatabase) -> None: assert 'file_size' in cols +def test_legacy_media_schema_repairs_required_refresh_columns(tmp_path: Path) -> None: + """Upgraded installs can have old library tables plus migration markers. + Startup must repair the columns full refresh writes later.""" + db_path = tmp_path / 'legacy_missing_media_columns.db' + conn = sqlite3.connect(str(db_path)) + cur = conn.cursor() + cur.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)") + cur.execute("INSERT INTO metadata (key, value) VALUES ('id_columns_migrated', 'true')") + cur.execute(""" + CREATE TABLE artists ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL + ) + """) + cur.execute(""" + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id TEXT NOT NULL, + title TEXT NOT NULL + ) + """) + cur.execute(""" + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT NOT NULL, + artist_id TEXT NOT NULL, + title TEXT NOT NULL, + file_path TEXT, + bitrate INTEGER + ) + """) + conn.commit() + conn.close() + + repaired = MusicDatabase(database_path=str(db_path)) + conn = repaired._get_connection() + cur = conn.cursor() + cur.execute("PRAGMA table_info(tracks)") + track_cols = {row[1] for row in cur.fetchall()} + cur.execute("PRAGMA table_info(albums)") + album_cols = {row[1] for row in cur.fetchall()} + conn.close() + + assert 'file_size' in track_cols + assert 'api_track_count' in album_cols + + def test_existing_tracks_have_null_file_size_after_migration(db: MusicDatabase) -> None: """Backward-compat: rows inserted via the OLD schema (no file_size) must still be readable, and querying file_size returns NULL — not diff --git a/tests/test_lyrics_reembed_from_sidecar.py b/tests/test_lyrics_reembed_from_sidecar.py new file mode 100644 index 00000000..498dda54 --- /dev/null +++ b/tests/test_lyrics_reembed_from_sidecar.py @@ -0,0 +1,220 @@ +"""Tests for re-embedding lyrics from an existing sidecar file. + +Discord report (Netti93): retag was clearing the LYRICS / USLT tag +without rewriting it. Cause was two-fold: + +1. `core/library/retag.py:execute_retag` never called + `generate_lrc_file` after `enhance_file_metadata`. The download + pipeline does — retag was inconsistent. +2. Even with the call added, `lyrics_client.create_lrc_file` used to + short-circuit when an .lrc / .txt sidecar already existed (the + typical retag case — sidecar moved alongside the audio file). + Pre-fix: returned True without re-embedding USLT. Post-fix: reads + the existing sidecar and re-embeds the USLT tag. +""" + +from __future__ import annotations + +import os +import tempfile +from unittest.mock import patch, MagicMock + +import pytest + + +@pytest.fixture +def fake_audio_file(tmp_path): + """Build a minimal FLAC file with no LYRICS tag.""" + fd, path = tempfile.mkstemp(suffix='.flac', dir=str(tmp_path)) + os.close(fd) + minimal = ( + b'fLaC' + + b'\x80\x00\x00\x22' + + b'\x00\x10\x00\x10' + + b'\x00\x00\x00\x00\x00\x00' + + b'\x0a\xc4\x42\xf0\x00\x00\x00\x00' + + b'\x00' * 16 + ) + with open(path, 'wb') as f: + f.write(minimal) + yield path + + +# ────────────────────────────────────────────────────────────────────── +# create_lrc_file — re-embed when sidecar present +# ────────────────────────────────────────────────────────────────────── + +def test_existing_lrc_sidecar_triggers_reembed(fake_audio_file): + """The exact retag scenario — sidecar already exists alongside the + audio file (moved during retag), USLT got cleared by enrichment. + Helper should read the sidecar and re-embed without hitting LRClib.""" + from core.lyrics_client import LyricsClient + + sidecar_path = os.path.splitext(fake_audio_file)[0] + '.lrc' + with open(sidecar_path, 'w', encoding='utf-8') as f: + f.write('[00:01.00]Test lyric line\n[00:05.00]Second line') + + client = LyricsClient() + client.api = MagicMock() # API stub — should NOT be called + client._embed_lyrics = MagicMock() + + result = client.create_lrc_file( + audio_file_path=fake_audio_file, + track_name='Test', + artist_name='Artist', + ) + + assert result is True + # API never hit — sidecar shortcut + client.api.get_lyrics.assert_not_called() + client.api.search_lyrics.assert_not_called() + # USLT was re-embedded + client._embed_lyrics.assert_called_once() + call_args = client._embed_lyrics.call_args + assert call_args.args[0] == fake_audio_file + assert 'Test lyric line' in call_args.args[1] + + +def test_existing_txt_sidecar_also_triggers_reembed(fake_audio_file): + """Same shape with .txt sidecar (plain lyrics, no timestamps).""" + from core.lyrics_client import LyricsClient + + sidecar_path = os.path.splitext(fake_audio_file)[0] + '.txt' + with open(sidecar_path, 'w', encoding='utf-8') as f: + f.write('Just plain lyrics no timestamps') + + client = LyricsClient() + client.api = MagicMock() + client._embed_lyrics = MagicMock() + + result = client.create_lrc_file( + audio_file_path=fake_audio_file, + track_name='T', artist_name='A', + ) + + assert result is True + client._embed_lyrics.assert_called_once_with( + fake_audio_file, 'Just plain lyrics no timestamps' + ) + + +def test_empty_sidecar_does_not_embed(fake_audio_file): + """Defensive — if the sidecar exists but is empty, don't write an + empty USLT tag.""" + from core.lyrics_client import LyricsClient + + sidecar_path = os.path.splitext(fake_audio_file)[0] + '.lrc' + with open(sidecar_path, 'w', encoding='utf-8') as f: + f.write(' \n ') # whitespace only + + client = LyricsClient() + client.api = MagicMock() + client._embed_lyrics = MagicMock() + + result = client.create_lrc_file( + audio_file_path=fake_audio_file, + track_name='T', artist_name='A', + ) + + assert result is True + client._embed_lyrics.assert_not_called() + + +def test_unreadable_sidecar_swallows_error_returns_true(fake_audio_file): + """If the sidecar is somehow unreadable, return True (don't try + LRClib again — the early-return contract holds), just skip the + embed silently.""" + from core.lyrics_client import LyricsClient + + sidecar_path = os.path.splitext(fake_audio_file)[0] + '.lrc' + with open(sidecar_path, 'wb') as f: + f.write(b'\xff\xfe\x00\x00') # invalid UTF-8 + + client = LyricsClient() + client.api = MagicMock() + client._embed_lyrics = MagicMock() + + result = client.create_lrc_file( + audio_file_path=fake_audio_file, + track_name='T', artist_name='A', + ) + + assert result is True + client.api.get_lyrics.assert_not_called() + + +def test_no_sidecar_falls_through_to_lrclib(fake_audio_file): + """No sidecar → original LRClib fetch path runs (download flow).""" + from core.lyrics_client import LyricsClient + + client = LyricsClient() + fake_lyrics = MagicMock() + fake_lyrics.synced_lyrics = '[00:01.00]synced from api' + fake_lyrics.plain_lyrics = None + client.api = MagicMock() + client.api.get_lyrics.return_value = None + client.api.search_lyrics.return_value = [fake_lyrics] + client._embed_lyrics = MagicMock() + + result = client.create_lrc_file( + audio_file_path=fake_audio_file, + track_name='T', artist_name='A', + ) + + assert result is True + client.api.search_lyrics.assert_called_once() + # Sidecar was created + lrc = os.path.splitext(fake_audio_file)[0] + '.lrc' + assert os.path.exists(lrc) + # And USLT was embedded + client._embed_lyrics.assert_called_once() + + +# ────────────────────────────────────────────────────────────────────── +# RetagDeps integration — generate_lrc_file is now wired +# ────────────────────────────────────────────────────────────────────── + +def test_retagdeps_accepts_generate_lrc_file_field(): + from core.library.retag import RetagDeps + + # Mock the required + optional deps with do-nothing callables + deps = RetagDeps( + config_manager=MagicMock(), + retag_lock=MagicMock(), + spotify_client=MagicMock(), + get_audio_quality_string=lambda *a: '', + enhance_file_metadata=lambda *a: True, + build_final_path_for_track=lambda *a: ('', ''), + safe_move_file=lambda *a: None, + cleanup_empty_directories=lambda *a: None, + download_cover_art=lambda *a: None, + docker_resolve_path=lambda x: x, + _get_retag_state=lambda: {}, + _set_retag_state=lambda v: None, + get_database=lambda: MagicMock(), + generate_lrc_file=lambda *a: True, + ) + assert callable(deps.generate_lrc_file) + + +def test_retagdeps_generate_lrc_file_optional_for_backward_compat(): + """Tests that built RetagDeps without the new field don't break.""" + from core.library.retag import RetagDeps + + deps = RetagDeps( + config_manager=MagicMock(), + retag_lock=MagicMock(), + spotify_client=MagicMock(), + get_audio_quality_string=lambda *a: '', + enhance_file_metadata=lambda *a: True, + build_final_path_for_track=lambda *a: ('', ''), + safe_move_file=lambda *a: None, + cleanup_empty_directories=lambda *a: None, + download_cover_art=lambda *a: None, + docker_resolve_path=lambda x: x, + _get_retag_state=lambda: {}, + _set_retag_state=lambda v: None, + get_database=lambda: MagicMock(), + ) + # Field defaults to None — no crash on construction. + assert deps.generate_lrc_file is None diff --git a/tests/test_manual_library_match.py b/tests/test_manual_library_match.py new file mode 100644 index 00000000..7804eb25 --- /dev/null +++ b/tests/test_manual_library_match.py @@ -0,0 +1,440 @@ +"""Tests for core/library/manual_library_match.py and DB methods.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from core.library import manual_library_match as mlm +from database.music_database import MusicDatabase + + +@pytest.fixture +def db(tmp_path): + return MusicDatabase(str(tmp_path / "music.db")) + + +# --------------------------------------------------------------------------- +# DB-layer tests +# --------------------------------------------------------------------------- + +def test_save_and_get_roundtrip(db): + ok = db.save_manual_library_match(1, "spotify", "track-abc", 42, + source_title="HUMBLE.", source_artist="Kendrick Lamar", + source_album="DAMN.", server_source="") + assert ok is True + row = db.get_manual_library_match(1, "spotify", "track-abc") + assert row is not None + assert row["library_track_id"] == 42 + assert row["source_title"] == "HUMBLE." + assert row["source_artist"] == "Kendrick Lamar" + + +def test_save_upserts_existing(db): + db.save_manual_library_match(1, "spotify", "track-abc", 42) + db.save_manual_library_match(1, "spotify", "track-abc", 99, source_title="Updated") + row = db.get_manual_library_match(1, "spotify", "track-abc") + assert row["library_track_id"] == 99 + assert row["source_title"] == "Updated" + + +def test_delete_by_id(db): + db.save_manual_library_match(1, "spotify", "track-abc", 42) + row = db.get_manual_library_match(1, "spotify", "track-abc") + assert row is not None + ok = db.delete_manual_library_match(row["id"], 1) + assert ok is True + assert db.get_manual_library_match(1, "spotify", "track-abc") is None + + +def test_list_matches_scoped_to_profile(db): + db.save_manual_library_match(1, "spotify", "t1", 10) + db.save_manual_library_match(1, "spotify", "t2", 20) + rows = db.list_manual_library_matches(1) + assert len(rows) == 2 + # Ordered by updated_at DESC — most recent first + ids = {r["library_track_id"] for r in rows} + assert ids == {10, 20} + + +def test_profile_isolation(db): + db.save_manual_library_match(1, "spotify", "track-abc", 10) + db.save_manual_library_match(2, "spotify", "track-abc", 20) + assert db.get_manual_library_match(1, "spotify", "track-abc")["library_track_id"] == 10 + assert db.get_manual_library_match(2, "spotify", "track-abc")["library_track_id"] == 20 + assert len(db.list_manual_library_matches(1)) == 1 + assert len(db.list_manual_library_matches(2)) == 1 + + +def test_server_source_isolation(db): + db.save_manual_library_match(1, "spotify", "track-abc", 10, server_source="plex") + db.save_manual_library_match(1, "spotify", "track-abc", 20, server_source="jellyfin") + assert db.get_manual_library_match(1, "spotify", "track-abc", "plex")["library_track_id"] == 10 + assert db.get_manual_library_match(1, "spotify", "track-abc", "jellyfin")["library_track_id"] == 20 + + +def test_get_returns_none_when_absent(db): + assert db.get_manual_library_match(1, "spotify", "nonexistent") is None + + +def test_get_match_for_track_falls_back_across_source_labels(db): + db.save_manual_library_match( + 1, + "mirrored", + "track-abc", + 42, + source_title="Coffee Break", + source_artist="Zeds Dead", + ) + + row = mlm.get_match_for_track( + db, + 1, + { + "id": "track-abc", + "name": "Coffee Break", + "artists": [{"name": "Zeds Dead"}], + "provider": "wishlist", + }, + default_source="wishlist", + ) + + assert row is not None + assert row["library_track_id"] == 42 + + +def test_get_match_for_track_falls_back_to_source_title_artist(db): + db.save_manual_library_match( + 1, + "mirrored", + "old-id", + 42, + source_title="Coffee Break", + source_artist="Zeds Dead", + ) + + row = mlm.get_match_for_track( + db, + 1, + { + "id": "new-id", + "name": "Coffee Break", + "artists": [{"name": "Zeds Dead"}], + "provider": "musicbrainz", + }, + default_source="wishlist", + ) + + assert row is not None + assert row["library_track_id"] == 42 + + +def test_add_to_wishlist_skips_manual_matched_track(db): + db.save_manual_library_match(1, "spotify", "track-abc", 42) + + ok = db.add_to_wishlist( + track_data={ + "id": "track-abc", + "name": "HUMBLE.", + "artists": [{"name": "Kendrick Lamar"}], + "album": {"name": "DAMN."}, + "provider": "spotify", + }, + failure_reason="Download failed", + profile_id=1, + ) + + assert ok is True + assert db.get_wishlist_tracks(profile_id=1) == [] + + +def test_add_to_wishlist_skips_manual_match_saved_from_mirrored_source(db): + db.save_manual_library_match( + 1, + "mirrored", + "track-abc", + 42, + source_title="Coffee Break", + source_artist="Zeds Dead", + ) + + ok = db.add_to_wishlist( + track_data={ + "id": "track-abc", + "name": "Coffee Break", + "artists": [{"name": "Zeds Dead"}], + "album": {"name": "Coffee Break"}, + "provider": "wishlist", + }, + failure_reason="Download failed", + profile_id=1, + ) + + assert ok is True + assert db.get_wishlist_tracks(profile_id=1) == [] + + +def test_get_match_returns_none_when_db_lacks_manual_match_method(): + class _MinimalDB: + pass + + assert mlm.get_match(_MinimalDB(), 1, "spotify", "track-abc") is None + + +# --------------------------------------------------------------------------- +# Service-layer tests +# --------------------------------------------------------------------------- + +def test_service_save_and_get(db): + mlm.save_match(db, 1, "spotify", "t1", 42, source_title="Song A") + row = mlm.get_match(db, 1, "spotify", "t1") + assert row is not None + assert row["library_track_id"] == 42 + + +def test_service_delete(db): + mlm.save_match(db, 1, "spotify", "t1", 42) + row = mlm.get_match(db, 1, "spotify", "t1") + mlm.delete_match(db, row["id"], 1) + assert mlm.get_match(db, 1, "spotify", "t1") is None + + +def test_service_list_enriches(db): + mlm.save_match(db, 1, "spotify", "t1", 42, source_title="Song A", source_artist="Artist X") + with patch.object(db, "api_get_tracks_by_ids", return_value=[{"title": "Song A", "artist_name": "Artist X", "album_title": "Album Z", "file_path": "/music/a.flac", "bitrate": 320}]): + matches = mlm.list_matches(db, 1) + assert len(matches) == 1 + assert matches[0]["library_title"] == "Song A" + + +def test_search_library_candidates(db): + with patch.object(db, "api_search_tracks", return_value=[{"id": 1, "title": "HUMBLE.", "artist_name": "Kendrick Lamar"}]) as mock_search: + results = mlm.search_library_candidates(db, "HUMBLE") + # searches by title AND artist, deduped + assert mock_search.call_count == 2 + assert len(results) == 1 + assert results[0]["title"] == "HUMBLE." + + +def test_search_source_candidates_empty_query(db): + results = mlm.search_source_candidates(db, "", 1) + assert results == [] + + +# --------------------------------------------------------------------------- +# Integration: wishlist processing +# --------------------------------------------------------------------------- + +def test_wishlist_skips_manual_matched_track(): + """Manual match causes track to be removed from wishlist without fuzzy check.""" + track = { + "name": "HUMBLE.", + "artists": [{"name": "Kendrick Lamar"}], + "spotify_track_id": "spotify-track-123", + } + + mock_wishlist_svc = MagicMock() + mock_wishlist_svc.get_wishlist_tracks_for_download.return_value = [track] + mock_wishlist_svc.mark_track_download_result.return_value = True + + mock_profiles_db = MagicMock() + mock_profiles_db.get_all_profiles.return_value = [{"id": 1}] + + mock_music_db = MagicMock() + mock_music_db.check_track_exists = MagicMock() + + with patch("core.library.manual_library_match.get_match_for_track", return_value={"id": 1, "library_track_id": 42}): + from core.wishlist.processing import remove_tracks_already_in_library + removed = remove_tracks_already_in_library( + mock_wishlist_svc, + mock_profiles_db, + mock_music_db, + active_server="plex", + ) + + assert removed == 1 + mock_wishlist_svc.mark_track_download_result.assert_called_once_with("spotify-track-123", success=True) + mock_music_db.check_track_exists.assert_not_called() + + +def test_wishlist_falls_through_when_no_match(): + """No manual match → fuzzy path runs normally.""" + track = { + "name": "HUMBLE.", + "artists": [{"name": "Kendrick Lamar"}], + "spotify_track_id": "spotify-track-123", + } + + mock_wishlist_svc = MagicMock() + mock_wishlist_svc.get_wishlist_tracks_for_download.return_value = [track] + mock_wishlist_svc.mark_track_download_result.return_value = False + + mock_profiles_db = MagicMock() + mock_profiles_db.get_all_profiles.return_value = [{"id": 1}] + + mock_music_db = MagicMock() + mock_music_db.check_track_exists.return_value = (None, 0.0) + + with patch("core.library.manual_library_match.get_match_for_track", return_value=None): + from core.wishlist.processing import remove_tracks_already_in_library + removed = remove_tracks_already_in_library( + mock_wishlist_svc, + mock_profiles_db, + mock_music_db, + active_server="plex", + ) + + assert removed == 0 + mock_music_db.check_track_exists.assert_called() + + +# --------------------------------------------------------------------------- +# Integration: downloads/master analysis loop +# --------------------------------------------------------------------------- + +def test_master_analysis_marks_found(): + """Manual match causes the real analysis loop to mark track found=True.""" + from core.downloads.master import MasterDeps, run_full_missing_tracks_process + from core.runtime_state import download_batches, tasks_lock + import threading + + batch_id = "test-batch-real-123" + track_data = {"name": "HUMBLE.", "artists": [], "id": "spotify-track-abc"} + + with tasks_lock: + download_batches[batch_id] = { + "phase": "analysis", + "analysis_total": 1, + "analysis_processed": 0, + "force_download_all": False, + "is_album_download": False, + "album_context": None, + "artist_context": None, + "profile_id": 1, + "batch_source": "spotify", + "wing_it": False, + "playlist_folder_mode": False, + "queue": [], + "active_count": 0, + "max_concurrent": 1, + "queue_index": 0, + "permanently_failed_tracks": [], + "cancelled_tracks": set(), + "analysis_results": [], + } + + mock_db = MagicMock() + mock_db.check_track_exists.return_value = (None, 0.0) + mock_db.update_sync_history_completion = MagicMock() + + mock_deps = MagicMock() + mock_deps.config_manager.get.return_value = False + mock_deps.config_manager.get_active_media_server.return_value = "plex" + mock_deps.mb_worker = None + mock_deps.mb_release_cache = {} + mock_deps.mb_release_cache_lock = threading.Lock() + mock_deps.mb_release_detail_cache = {} + mock_deps.mb_release_detail_cache_lock = threading.Lock() + mock_deps.normalize_album_cache_key = lambda x: x.lower().strip() + mock_deps.check_and_remove_track_from_wishlist_by_metadata = MagicMock() + mock_deps.is_explicit_blocked = MagicMock(return_value=False) + mock_deps.youtube_playlist_states = {} + mock_deps.tidal_discovery_states = {} + mock_deps.deezer_discovery_states = {} + mock_deps.spotify_public_discovery_states = {} + mock_deps.missing_download_executor = MagicMock() + mock_deps.process_failed_tracks_to_wishlist_exact_with_auto_completion = MagicMock() + mock_deps.source_reuse_logger = MagicMock() + mock_deps.download_monitor = MagicMock() + mock_deps.start_next_batch_of_downloads = MagicMock() + mock_deps.reset_wishlist_auto_processing = MagicMock() + + with patch("core.library.manual_library_match.get_match_for_track", return_value={"id": 1, "library_track_id": 42}), \ + patch("database.music_database.MusicDatabase", return_value=mock_db): + run_full_missing_tracks_process(batch_id, "playlist-1", [track_data], mock_deps) + + with tasks_lock: + results = download_batches.get(batch_id, {}).get("analysis_results", []) + download_batches.pop(batch_id, None) + + assert len(results) == 1 + assert results[0]["found"] is True + assert results[0]["match_reason"] == "manual_library_match" + mock_deps.check_and_remove_track_from_wishlist_by_metadata.assert_called_once_with(track_data) + + +def test_master_analysis_manual_match_wins_over_internal_force_download(): + """Manual match overrides internal force_download_all used by wishlist batches.""" + from core.downloads.master import run_full_missing_tracks_process + from core.runtime_state import download_batches, tasks_lock + import threading + + batch_id = "test-batch-force-456" + track_data = {"name": "HUMBLE.", "artists": [], "id": "spotify-track-abc"} + + with tasks_lock: + download_batches[batch_id] = { + "phase": "analysis", + "analysis_total": 1, + "analysis_processed": 0, + "force_download_all": True, # would normally bypass DB check and queue download + "ignore_manual_matches": False, + "is_album_download": False, + "album_context": None, + "artist_context": None, + "profile_id": 1, + "batch_source": "spotify", + "wing_it": False, + "playlist_folder_mode": False, + "queue": [], + "active_count": 0, + "max_concurrent": 1, + "queue_index": 0, + "permanently_failed_tracks": [], + "cancelled_tracks": set(), + "analysis_results": [], + } + + mock_db = MagicMock() + mock_db.check_track_exists.return_value = (None, 0.0) + mock_db.update_sync_history_completion = MagicMock() + + mock_deps = MagicMock() + mock_deps.config_manager.get.return_value = False + mock_deps.config_manager.get_active_media_server.return_value = "plex" + mock_deps.mb_worker = None + mock_deps.mb_release_cache = {} + mock_deps.mb_release_cache_lock = threading.Lock() + mock_deps.mb_release_detail_cache = {} + mock_deps.mb_release_detail_cache_lock = threading.Lock() + mock_deps.normalize_album_cache_key = lambda x: x.lower().strip() + mock_deps.check_and_remove_track_from_wishlist_by_metadata = MagicMock() + mock_deps.is_explicit_blocked = MagicMock(return_value=False) + mock_deps.youtube_playlist_states = {} + mock_deps.tidal_discovery_states = {} + mock_deps.deezer_discovery_states = {} + mock_deps.spotify_public_discovery_states = {} + mock_deps.missing_download_executor = MagicMock() + mock_deps.process_failed_tracks_to_wishlist_exact_with_auto_completion = MagicMock() + mock_deps.source_reuse_logger = MagicMock() + mock_deps.download_monitor = MagicMock() + mock_deps.start_next_batch_of_downloads = MagicMock() + mock_deps.reset_wishlist_auto_processing = MagicMock() + + with patch("core.library.manual_library_match.get_match_for_track", return_value={"id": 1, "library_track_id": 42}), \ + patch("database.music_database.MusicDatabase", return_value=mock_db): + run_full_missing_tracks_process(batch_id, "playlist-1", [track_data], mock_deps) + + with tasks_lock: + results = download_batches.get(batch_id, {}).get("analysis_results", []) + queue = download_batches.get(batch_id, {}).get("queue", []) + download_batches.pop(batch_id, None) + + assert len(results) == 1 + assert results[0]["found"] is True + assert results[0]["match_reason"] == "manual_library_match" + # Track must NOT enter the download queue despite internal force_download_all=True + assert queue == [] + mock_deps.check_and_remove_track_from_wishlist_by_metadata.assert_called_once_with(track_data) diff --git a/tests/test_manual_pick_no_auto_retry.py b/tests/test_manual_pick_no_auto_retry.py index c5b23cfe..9c70f0f8 100644 --- a/tests/test_manual_pick_no_auto_retry.py +++ b/tests/test_manual_pick_no_auto_retry.py @@ -93,3 +93,204 @@ def test_manual_pick_skips_retry_on_errored_state(fake_monitor): assert task['status'] == 'downloading' +def test_monitor_waits_for_post_processing_before_batch_success(monkeypatch): + """Engine completion is not the same as a successful import. + + The monitor should start post-processing when slskd reports a completed + transfer, but the post-processing worker must be the only code path that + reports final success/failure to the batch lifecycle. + """ + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + completions = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr( + dm, + '_on_download_completed', + lambda batch_id, task_id, success: completions.append((batch_id, task_id, success)), + ) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'Test Track'}, + 'username': 'Pinasound', + 'filename': r'@@tmllb\Music\Album\01. Track.flac', + 'status': 'downloading', + 'download_id': 'download-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + r'Pinasound::@@tmllb\Music\Album\01. Track.flac': { + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert completions == [] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + + +def test_monitor_matches_release_download_by_id_when_filename_changes(monkeypatch): + """Torrent/usenet rows can expose the completed audio filename, not + the original indexer URL/title stored on the task. The monitor must + still claim the completed release by stable download_id. + """ + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'Ran To Atlanta'}, + 'username': 'torrent', + 'filename': 'http://prowlarr/download?id=123||Drake - ICEMAN', + 'status': 'downloading', + 'download_id': 'torrent-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + 'download_id::torrent-1': { + 'id': 'torrent-1', + 'username': 'torrent', + 'filename': '01. Drake - Make Them Cry.flac', + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + + +def test_monitor_recovers_premature_failed_release_download(monkeypatch): + monkeypatch.setattr(dm, '_make_context_key', lambda u, f: f"{u}::{f}") + monkeypatch.setattr(dm.WebUIDownloadMonitor, '_validate_worker_counts', lambda self: None) + + submitted = [] + + class FakeExecutor: + def submit(self, func, task_id, batch_id): + submitted.append((func, task_id, batch_id)) + + def fake_post_processing_worker(task_id, batch_id): + return None + + monkeypatch.setattr(dm, 'missing_download_executor', FakeExecutor()) + monkeypatch.setattr(dm, '_run_post_processing_worker', fake_post_processing_worker) + monkeypatch.setattr(dm, '_on_download_completed', lambda *args: None) + + with dm.tasks_lock: + previous_tasks = dict(dm.download_tasks) + previous_batches = dict(dm.download_batches) + dm.download_tasks.clear() + dm.download_batches.clear() + dm.download_tasks['task-1'] = { + 'track_info': {'name': 'DAISIES'}, + 'username': 'torrent', + 'filename': 'http://prowlarr/download?id=123||Justin Bieber - Swag', + 'status': 'failed', + 'download_id': 'torrent-1', + 'status_change_time': time.time(), + } + dm.download_batches['batch-1'] = {'queue': ['task-1']} + + try: + monitor = dm.WebUIDownloadMonitor() + monitor.monitoring = True + monitor.monitored_batches.add('batch-1') + monkeypatch.setattr( + monitor, + '_get_live_transfers', + lambda: { + 'download_id::torrent-1': { + 'id': 'torrent-1', + 'username': 'torrent', + 'filename': '02. Justin Bieber - DAISIES.flac', + 'state': 'Completed, Succeeded', + 'size': 100, + 'bytesTransferred': 100, + } + }, + ) + + monitor._check_all_downloads() + + assert submitted == [(fake_post_processing_worker, 'task-1', 'batch-1')] + assert dm.download_tasks['task-1']['status'] == 'post_processing' + finally: + with dm.tasks_lock: + dm.download_tasks.clear() + dm.download_tasks.update(previous_tasks) + dm.download_batches.clear() + dm.download_batches.update(previous_batches) + diff --git a/tests/test_orphan_file_detector.py b/tests/test_orphan_file_detector.py new file mode 100644 index 00000000..02ba7150 --- /dev/null +++ b/tests/test_orphan_file_detector.py @@ -0,0 +1,81 @@ +"""Regression tests for the orphan file detector.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from core.repair_jobs.base import JobContext +from core.repair_jobs.orphan_file_detector import OrphanFileDetectorJob + + +class _DB: + def __init__(self, path: Path) -> None: + self.path = path + + def _get_connection(self): + return sqlite3.connect(self.path) + + +def _seed_library(db_path: Path) -> None: + conn = sqlite3.connect(db_path) + try: + conn.executescript( + """ + CREATE TABLE artists ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL + ); + CREATE TABLE albums ( + id INTEGER PRIMARY KEY, + artist_id INTEGER NOT NULL, + title TEXT NOT NULL + ); + CREATE TABLE tracks ( + id INTEGER PRIMARY KEY, + album_id INTEGER NOT NULL, + artist_id INTEGER NOT NULL, + title TEXT NOT NULL, + file_path TEXT + ); + INSERT INTO artists (id, name) VALUES + (1, 'Clouddead89'), + (2, 'Featured Artist'); + INSERT INTO albums (id, artist_id, title) VALUES + (10, 1, 'Perfect Match Error'); + INSERT INTO tracks (id, album_id, artist_id, title, file_path) VALUES + (100, 10, 2, 'Perfect Match', '/old/prefix/elsewhere.mp3'); + """ + ) + conn.commit() + finally: + conn.close() + + +def test_orphan_detector_accepts_picard_albumartist_folder_match(tmp_path: Path) -> None: + """Picard paths use albumartist/album (year)/track - title. + + Even when the DB track artist is a featured artist, the album artist + folder should be enough to recognize the file as tracked. + """ + db_path = tmp_path / "library.sqlite" + _seed_library(db_path) + + transfer = tmp_path / "Clouddead89" / "Perfect Match Error (2026)" + transfer.mkdir(parents=True) + audio_path = transfer / "01 - Perfect Match.mp3" + audio_path.write_bytes(b"not a real mp3; filename fallback handles this") + + findings = [] + context = JobContext( + db=_DB(db_path), + transfer_folder=str(tmp_path), + config_manager=None, + create_finding=lambda **kwargs: findings.append(kwargs) or True, + ) + + result = OrphanFileDetectorJob().scan(context) + + assert result.scanned == 1 + assert result.findings_created == 0 + assert findings == [] diff --git a/tests/test_personalized_api.py b/tests/test_personalized_api.py new file mode 100644 index 00000000..7b7ef613 --- /dev/null +++ b/tests/test_personalized_api.py @@ -0,0 +1,181 @@ +"""Boundary tests for `core.personalized.api` handler functions. + +These are pure-function dispatchers — they take a manager + ids, +return a JSON-serializable dict. No Flask required, no real DB. +The Flask wiring in `web_server.py` adds `jsonify` + URL routing +on top. +""" + +from __future__ import annotations + +import sqlite3 +from types import SimpleNamespace +from typing import Any, List + +import pytest + +from core.personalized import api as _api +from core.personalized.manager import PersonalizedPlaylistManager +from core.personalized.specs import PlaylistKindRegistry, PlaylistKindSpec +from core.personalized.types import PlaylistConfig, Track +from database.personalized_schema import ensure_personalized_schema + + +class _FakeDB: + def __init__(self, path): + self.path = path + + def _get_connection(self): + c = sqlite3.connect(self.path) + c.row_factory = sqlite3.Row + return c + + +@pytest.fixture +def db(tmp_path): + p = str(tmp_path / 't.db') + conn = sqlite3.connect(p) + ensure_personalized_schema(conn) + conn.commit() + conn.close() + return _FakeDB(p) + + +@pytest.fixture +def registry(): + return PlaylistKindRegistry() + + +@pytest.fixture +def manager(db, registry): + return PersonalizedPlaylistManager(db, deps=None, registry=registry) + + +def _register(reg, kind='hidden_gems', requires_variant=False, generator=None): + spec = PlaylistKindSpec( + kind=kind, name_template=kind.replace('_', ' ').title(), + description=f'Description for {kind}', + default_config=PlaylistConfig(limit=20), + generator=generator or (lambda *a, **k: []), + requires_variant=requires_variant, + tags=['test'], + ) + reg.register(spec) + return spec + + +# ─── list_kinds ────────────────────────────────────────────────────── + + +class TestListKinds: + def test_lists_every_registered_kind(self, registry): + _register(registry, kind='hidden_gems') + _register(registry, kind='time_machine', requires_variant=True) + out = _api.list_kinds(registry) + assert out['success'] is True + kinds = {k['kind'] for k in out['kinds']} + assert kinds == {'hidden_gems', 'time_machine'} + + def test_kind_metadata_shape(self, registry): + _register(registry, kind='hidden_gems') + out = _api.list_kinds(registry) + kind = out['kinds'][0] + assert kind['kind'] == 'hidden_gems' + assert kind['requires_variant'] is False + assert kind['tags'] == ['test'] + assert kind['default_config']['limit'] == 20 + + def test_empty_registry(self): + out = _api.list_kinds(PlaylistKindRegistry()) + assert out == {'success': True, 'kinds': []} + + +# ─── list_playlists ───────────────────────────────────────────────── + + +class TestListPlaylists: + def test_returns_empty_when_no_playlists(self, manager, registry): + _register(registry) + out = _api.list_playlists(manager, profile_id=1) + assert out == {'success': True, 'playlists': []} + + def test_serializes_playlist_record(self, manager, registry): + _register(registry) + manager.ensure_playlist('hidden_gems', '', 1) + out = _api.list_playlists(manager, profile_id=1) + assert out['success'] is True + assert len(out['playlists']) == 1 + pl = out['playlists'][0] + assert pl['kind'] == 'hidden_gems' + assert pl['variant'] == '' + assert pl['name'] == 'Hidden Gems' + assert pl['track_count'] == 0 + assert pl['config']['limit'] == 20 + + +# ─── get_playlist_with_tracks ─────────────────────────────────────── + + +class TestGetPlaylistWithTracks: + def test_auto_creates_on_first_get(self, manager, registry): + _register(registry) + out = _api.get_playlist_with_tracks(manager, 'hidden_gems', '', 1) + assert out['success'] is True + assert out['playlist']['kind'] == 'hidden_gems' + assert out['tracks'] == [] + + def test_returns_persisted_tracks(self, manager, registry): + gen_calls = [] + + def gen(deps, variant, config): + gen_calls.append(1) + return [Track(track_name='X', artist_name='Y', spotify_track_id='sp-1')] + + _register(registry, generator=gen) + manager.refresh_playlist('hidden_gems', '', 1) + out = _api.get_playlist_with_tracks(manager, 'hidden_gems', '', 1) + assert len(out['tracks']) == 1 + assert out['tracks'][0]['track_name'] == 'X' + assert out['tracks'][0]['spotify_track_id'] == 'sp-1' + + def test_unknown_kind_raises_value_error(self, manager): + with pytest.raises(ValueError): + _api.get_playlist_with_tracks(manager, 'nope', '', 1) + + +# ─── refresh_playlist ─────────────────────────────────────────────── + + +class TestRefreshPlaylist: + def test_refresh_runs_generator_and_returns_tracks(self, manager, registry): + _register(registry, generator=lambda *a, **k: [ + Track(track_name='T1', artist_name='A', spotify_track_id='1'), + Track(track_name='T2', artist_name='B', spotify_track_id='2'), + ]) + out = _api.refresh_playlist(manager, 'hidden_gems', '', 1) + assert out['success'] is True + assert len(out['tracks']) == 2 + assert out['playlist']['track_count'] == 2 + assert out['playlist']['last_generated_at'] is not None + + def test_config_overrides_passed_through(self, manager, registry): + captured = {} + + def gen(deps, variant, config): + captured['limit'] = config.limit + return [] + + _register(registry, generator=gen) + _api.refresh_playlist(manager, 'hidden_gems', '', 1, config_overrides={'limit': 99}) + assert captured['limit'] == 99 + + +# ─── update_config ────────────────────────────────────────────────── + + +class TestUpdateConfig: + def test_patches_config(self, manager, registry): + _register(registry) + out = _api.update_config(manager, 'hidden_gems', '', 1, {'limit': 75}) + assert out['success'] is True + assert out['playlist']['config']['limit'] == 75 diff --git a/tests/test_personalized_generators_curated.py b/tests/test_personalized_generators_curated.py new file mode 100644 index 00000000..f8df2c33 --- /dev/null +++ b/tests/test_personalized_generators_curated.py @@ -0,0 +1,316 @@ +"""Boundary tests for the curated / hybrid personalized generators +(`daily_mix`, `fresh_tape`, `archives`, `seasonal_mix`).""" + +from __future__ import annotations + +import sqlite3 +from types import SimpleNamespace +from typing import Any, List +from unittest.mock import MagicMock + +import pytest + +from core.personalized.generators import archives as _arch_mod +from core.personalized.generators import daily_mix as _dm_mod +from core.personalized.generators import fresh_tape as _ft_mod +from core.personalized.generators import seasonal_mix as _sm_mod +from core.personalized.specs import get_registry +from core.personalized.types import PlaylistConfig + + +# ─── daily_mix ─────────────────────────────────────────────────────── + + +class _DailyMixService: + """Stub PersonalizedPlaylistsService for daily_mix tests.""" + + GENRE_MAPPING = {} + + def __init__(self, top_genres=None, genre_tracks=None): + self._top = top_genres or [] + self._tracks = genre_tracks or {} + self.calls: List[dict] = [] + + def get_top_genres_from_library(self, limit): + self.calls.append({'method': 'get_top_genres_from_library', 'limit': limit}) + return self._top + + def get_genre_playlist(self, genre, limit, **kw): + self.calls.append({'method': 'get_genre_playlist', 'genre': genre, 'limit': limit}) + return self._tracks.get(genre, []) + + +class TestDailyMix: + def test_registered(self): + spec = get_registry().get('daily_mix') + assert spec is not None + assert spec.requires_variant is True + + def test_variant_resolver_returns_ranks(self): + spec = get_registry().get('daily_mix') + ranks = spec.variant_resolver(SimpleNamespace(service=_DailyMixService())) + assert ranks == ['1', '2', '3', '4'] + + def test_resolves_rank_to_top_genre(self): + svc = _DailyMixService( + top_genres=[('Rock', 100), ('Pop', 80), ('Jazz', 30)], + genre_tracks={'Rock': [{'track_name': 'R', 'artist_name': 'A'}]}, + ) + out = _dm_mod.generate(SimpleNamespace(service=svc), '1', PlaylistConfig(limit=10)) + assert len(out) == 1 + assert out[0].track_name == 'R' + # Service called for top-genre lookup + genre playlist. + assert {c['method'] for c in svc.calls} == { + 'get_top_genres_from_library', 'get_genre_playlist', + } + + def test_rank_beyond_top_returns_empty(self): + svc = _DailyMixService(top_genres=[('Rock', 100)]) # only 1 top genre + out = _dm_mod.generate(SimpleNamespace(service=svc), '4', PlaylistConfig()) + assert out == [] + + def test_invalid_variant_raises(self): + deps = SimpleNamespace(service=_DailyMixService()) + with pytest.raises(ValueError, match='must be a rank int'): + _dm_mod.generate(deps, 'abc', PlaylistConfig()) + + +# ─── fresh_tape / archives shared shape ───────────────────────────── + + +class _StubPoolTrack: + def __init__(self, sid, name='T', artist='A', source='spotify'): + self.spotify_track_id = sid + self.itunes_track_id = None + self.deezer_track_id = None + self.track_name = name + self.artist_name = artist + self.album_name = 'Album' + self.album_cover_url = None + self.duration_ms = 200000 + self.popularity = 50 + self.track_data_json = None + self.source = source + + +class _CuratedDB: + def __init__(self, curated_ids=None, pool_tracks=None): + self.curated_ids = curated_ids or [] + self.pool_tracks = pool_tracks or [] + self.requested_keys: List[str] = [] + + def get_curated_playlist(self, key, profile_id=1): + self.requested_keys.append(key) + return list(self.curated_ids) + + def get_discovery_pool_tracks(self, **kwargs): + return list(self.pool_tracks) + + +def _curated_deps(db): + return SimpleNamespace( + database=db, + get_current_profile_id=lambda: 1, + get_active_discovery_source=lambda: 'spotify', + ) + + +class TestFreshTape: + def test_registered(self): + spec = get_registry().get('fresh_tape') + assert spec is not None + assert spec.requires_variant is False + assert spec.display_name('') == 'Fresh Tape' + + def test_returns_empty_when_no_curated_ids(self): + db = _CuratedDB(curated_ids=[]) + out = _ft_mod.generate(_curated_deps(db), '', PlaylistConfig()) + assert out == [] + + def test_hydrates_curated_ids_from_pool(self): + db = _CuratedDB( + curated_ids=['sp-1', 'sp-2', 'sp-missing'], + pool_tracks=[ + _StubPoolTrack('sp-1', name='Song1', artist='Artist1'), + _StubPoolTrack('sp-2', name='Song2', artist='Artist2'), + ], + ) + out = _ft_mod.generate(_curated_deps(db), '', PlaylistConfig()) + # Missing IDs silently skipped; order preserved. + assert [t.track_name for t in out] == ['Song1', 'Song2'] + + def test_tries_source_specific_then_fallback_key(self): + # First lookup (source-specific) returns []; second (generic) returns IDs. + class _DB: + def __init__(self): + self.calls = [] + self.responses = { + 'release_radar_spotify': [], + 'release_radar': ['sp-1'], + } + + def get_curated_playlist(self, key, profile_id=1): + self.calls.append(key) + return self.responses.get(key, []) + + def get_discovery_pool_tracks(self, **kw): + return [_StubPoolTrack('sp-1', name='Hit')] + + db = _DB() + out = _ft_mod.generate(_curated_deps(db), '', PlaylistConfig()) + assert db.calls == ['release_radar_spotify', 'release_radar'] + assert len(out) == 1 + + def test_respects_limit(self): + db = _CuratedDB( + curated_ids=[f'sp-{i}' for i in range(20)], + pool_tracks=[_StubPoolTrack(f'sp-{i}', name=f'T{i}') for i in range(20)], + ) + out = _ft_mod.generate(_curated_deps(db), '', PlaylistConfig(limit=5)) + assert len(out) == 5 + + def test_missing_database_dep_raises(self): + with pytest.raises(RuntimeError, match='missing `database`'): + _ft_mod.generate(SimpleNamespace(), '', PlaylistConfig()) + + +class TestArchives: + def test_registered(self): + spec = get_registry().get('archives') + assert spec is not None + assert spec.display_name('') == 'The Archives' + + def test_uses_discovery_weekly_curated_key(self): + db = _CuratedDB( + curated_ids=['sp-1'], + pool_tracks=[_StubPoolTrack('sp-1', name='Discover')], + ) + _arch_mod.generate(_curated_deps(db), '', PlaylistConfig()) + # Source-specific request fires first; fallback only fires + # when source-specific returns empty. Stub returns IDs on + # every call, so only the first key gets queried. + assert db.requested_keys[0] == 'discovery_weekly_spotify' + + +# ─── seasonal_mix ─────────────────────────────────────────────────── + + +class _SeasonalService: + def __init__(self, track_ids): + self.track_ids = track_ids + + def get_curated_seasonal_playlist(self, season_key, source=None): + return list(self.track_ids) + + +@pytest.fixture +def seasonal_db(tmp_path): + """Real sqlite DB with seasonal_tracks rows for hydration.""" + p = str(tmp_path / 'seasonal.db') + conn = sqlite3.connect(p) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(""" + CREATE TABLE seasonal_tracks ( + id INTEGER PRIMARY KEY, + season_key TEXT, source TEXT, + spotify_track_id TEXT, track_name TEXT, artist_name TEXT, + album_name TEXT, album_cover_url TEXT, + duration_ms INTEGER, popularity INTEGER, track_data_json TEXT + ) + """) + seed = [ + ('halloween', 'spotify', 'sp-1', 'Spooky', 'Ghost Band', 'Album1', None, 200000, 80, '{"id":"sp-1"}'), + ('halloween', 'spotify', 'sp-2', 'Haunted', 'Ghost Band', 'Album2', None, 210000, 70, None), + ('halloween', 'spotify', 'sp-extra', 'Extra', 'Other', 'Album3', None, 200000, 60, None), + ] + cursor.executemany(""" + INSERT INTO seasonal_tracks + (season_key, source, spotify_track_id, track_name, artist_name, + album_name, album_cover_url, duration_ms, popularity, track_data_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, seed) + conn.commit() + conn.close() + + class _DB: + def __init__(self, path): self.path = path + def _get_connection(self): + c = sqlite3.connect(self.path) + c.row_factory = sqlite3.Row + return c + return _DB(p) + + +class TestSeasonalMix: + def test_registered(self): + spec = get_registry().get('seasonal_mix') + assert spec is not None + assert spec.requires_variant is True + + def test_variant_resolver_returns_seasons(self): + spec = get_registry().get('seasonal_mix') + seasons = spec.variant_resolver(None) + assert 'halloween' in seasons + assert 'christmas' in seasons + + def test_no_variant_raises(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-1']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + with pytest.raises(ValueError, match='requires a season variant'): + _sm_mod.generate(deps, '', PlaylistConfig()) + + def test_hydrates_curated_ids_in_order(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-2', 'sp-1']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig()) + assert [t.track_name for t in out] == ['Haunted', 'Spooky'] + + def test_missing_track_id_silently_skipped(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-1', 'sp-not-in-db']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig()) + assert len(out) == 1 + assert out[0].track_name == 'Spooky' + + def test_track_data_json_round_trips(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-1']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig()) + # sp-1 had JSON; sp-2 had None. + assert out[0].track_data_json == {'id': 'sp-1'} + + def test_empty_curated_returns_empty(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService([]), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig()) + assert out == [] + + def test_respects_limit(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-1', 'sp-2', 'sp-extra']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig(limit=2)) + assert len(out) == 2 + + def test_missing_seasonal_service_raises(self): + deps = SimpleNamespace(database=object()) + with pytest.raises(RuntimeError, match='missing `seasonal_service`'): + _sm_mod.generate(deps, 'halloween', PlaylistConfig()) diff --git a/tests/test_personalized_generators_singletons.py b/tests/test_personalized_generators_singletons.py new file mode 100644 index 00000000..6f39fd5a --- /dev/null +++ b/tests/test_personalized_generators_singletons.py @@ -0,0 +1,147 @@ +"""Boundary tests for the singleton-kind personalized generators +(`hidden_gems`, `discovery_shuffle`, `popular_picks`). + +Each generator wraps the legacy +``PersonalizedPlaylistsService`` method 1:1, so the tests pin: +- registration side-effect at import +- generator forwards `config.limit` correctly +- empty / None / non-dict service output → [] +- tracks coerced through `Track.from_dict` +- missing service in deps raises a clear error""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any, List + +import pytest + +# Importing each generator triggers registration as a side-effect. +from core.personalized.generators import discovery_shuffle as _ds_mod +from core.personalized.generators import hidden_gems as _hg_mod +from core.personalized.generators import popular_picks as _pp_mod +from core.personalized.specs import get_registry +from core.personalized.types import PlaylistConfig + + +class _StubService: + """Records every call so tests can assert on dispatched limits.""" + + def __init__(self, return_value=None): + self.calls: List[dict] = [] + self.return_value = return_value if return_value is not None else [] + + def get_hidden_gems(self, limit): + self.calls.append({'method': 'get_hidden_gems', 'limit': limit}) + return self.return_value + + def get_discovery_shuffle(self, limit): + self.calls.append({'method': 'get_discovery_shuffle', 'limit': limit}) + return self.return_value + + def get_popular_picks(self, limit): + self.calls.append({'method': 'get_popular_picks', 'limit': limit}) + return self.return_value + + +def _deps(svc): + return SimpleNamespace(service=svc) + + +# ─── registration ──────────────────────────────────────────────────── + + +class TestRegistration: + def test_hidden_gems_registered(self): + spec = get_registry().get('hidden_gems') + assert spec is not None + assert spec.kind == 'hidden_gems' + assert spec.requires_variant is False + assert spec.default_config.limit == 50 + + def test_discovery_shuffle_registered(self): + spec = get_registry().get('discovery_shuffle') + assert spec is not None + assert spec.requires_variant is False + + def test_popular_picks_registered(self): + spec = get_registry().get('popular_picks') + assert spec is not None + assert spec.requires_variant is False + + def test_display_names(self): + assert get_registry().get('hidden_gems').display_name('') == 'Hidden Gems' + assert get_registry().get('discovery_shuffle').display_name('') == 'Discovery Shuffle' + assert get_registry().get('popular_picks').display_name('') == 'Popular Picks' + + +# ─── generator dispatch ────────────────────────────────────────────── + + +class TestHiddenGemsGenerator: + def test_forwards_limit(self): + svc = _StubService() + _hg_mod.generate(_deps(svc), '', PlaylistConfig(limit=75)) + assert svc.calls == [{'method': 'get_hidden_gems', 'limit': 75}] + + def test_uses_default_limit_when_config_default(self): + svc = _StubService() + _hg_mod.generate(_deps(svc), '', PlaylistConfig()) + assert svc.calls[0]['limit'] == 50 + + def test_coerces_tracks(self): + svc = _StubService(return_value=[ + {'track_name': 'A', 'artist_name': 'X', 'spotify_track_id': 'sp-1'}, + {'track_name': 'B', 'artist_name': 'Y', 'spotify_track_id': 'sp-2'}, + ]) + out = _hg_mod.generate(_deps(svc), '', PlaylistConfig()) + assert len(out) == 2 + assert out[0].track_name == 'A' + assert out[0].spotify_track_id == 'sp-1' + + def test_empty_service_output_returns_empty_list(self): + svc = _StubService(return_value=[]) + out = _hg_mod.generate(_deps(svc), '', PlaylistConfig()) + assert out == [] + + def test_none_service_output_returns_empty_list(self): + svc = _StubService(return_value=None) + out = _hg_mod.generate(_deps(svc), '', PlaylistConfig()) + assert out == [] + + +class TestDiscoveryShuffleGenerator: + def test_forwards_limit(self): + svc = _StubService() + _ds_mod.generate(_deps(svc), '', PlaylistConfig(limit=42)) + assert svc.calls == [{'method': 'get_discovery_shuffle', 'limit': 42}] + + def test_coerces_tracks(self): + svc = _StubService(return_value=[{'track_name': 'Z', 'artist_name': 'Q'}]) + out = _ds_mod.generate(_deps(svc), '', PlaylistConfig()) + assert out[0].track_name == 'Z' + + +class TestPopularPicksGenerator: + def test_forwards_limit(self): + svc = _StubService() + _pp_mod.generate(_deps(svc), '', PlaylistConfig(limit=10)) + assert svc.calls == [{'method': 'get_popular_picks', 'limit': 10}] + + +# ─── deps validation ───────────────────────────────────────────────── + + +class TestDepsValidation: + def test_missing_service_raises(self): + # No `service` attribute on deps. + deps = SimpleNamespace() + with pytest.raises(RuntimeError, match='missing `service`'): + _hg_mod.generate(deps, '', PlaylistConfig()) + + def test_dict_form_deps_accepted(self): + # generators._common.get_service tolerates dict deps too. + svc = _StubService() + out = _hg_mod.generate({'service': svc}, '', PlaylistConfig()) + assert isinstance(out, list) + assert svc.calls diff --git a/tests/test_personalized_generators_variants.py b/tests/test_personalized_generators_variants.py new file mode 100644 index 00000000..40eb04d6 --- /dev/null +++ b/tests/test_personalized_generators_variants.py @@ -0,0 +1,131 @@ +"""Boundary tests for variant-bearing personalized generators +(`time_machine` per decade, `genre_playlist` per genre). + +Each generator coerces a URL-safe variant string into the form the +legacy service expects, then forwards. Tests pin the variant +parsing + service dispatch + variant_resolver listing.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import List + +import pytest + +from core.personalized.generators import genre_playlist as _gp_mod +from core.personalized.generators import time_machine as _tm_mod +from core.personalized.specs import get_registry +from core.personalized.types import PlaylistConfig + + +class _StubService: + GENRE_MAPPING = { + 'Electronic/Dance': ['house', 'techno'], + 'Hip Hop/Rap': ['hip hop', 'rap'], + 'Rock': ['rock', 'punk'], + } + + def __init__(self): + self.calls: List[dict] = [] + + def get_decade_playlist(self, decade, limit, **kw): + self.calls.append({'method': 'get_decade_playlist', 'decade': decade, 'limit': limit}) + return [{'track_name': f'D{decade}', 'artist_name': 'A'}] + + def get_genre_playlist(self, genre, limit, **kw): + self.calls.append({'method': 'get_genre_playlist', 'genre': genre, 'limit': limit}) + return [{'track_name': f'G{genre}', 'artist_name': 'A'}] + + +def _deps(): + return SimpleNamespace(service=_StubService()) + + +# ─── time_machine ─────────────────────────────────────────────────── + + +class TestTimeMachine: + def test_registered(self): + spec = get_registry().get('time_machine') + assert spec is not None + assert spec.requires_variant is True + assert spec.variant_resolver is not None + + def test_variant_resolver_returns_decades(self): + spec = get_registry().get('time_machine') + decades = spec.variant_resolver(_deps()) + assert '1980s' in decades + assert '2020s' in decades + # All decades should be 4-digit + 's' + for d in decades: + assert d.endswith('s') + assert d[:-1].isdigit() + + def test_decade_label_to_year(self): + deps = _deps() + _tm_mod.generate(deps, '1980s', PlaylistConfig(limit=20)) + assert deps.service.calls == [ + {'method': 'get_decade_playlist', 'decade': 1980, 'limit': 20} + ] + + def test_invalid_variant_raises(self): + deps = _deps() + with pytest.raises(ValueError, match='not a decade label'): + _tm_mod.generate(deps, 'banana', PlaylistConfig()) + + def test_out_of_range_year_raises(self): + deps = _deps() + with pytest.raises(ValueError, match='out of range'): + _tm_mod.generate(deps, '1500s', PlaylistConfig()) + + def test_tolerates_no_s_suffix(self): + deps = _deps() + _tm_mod.generate(deps, '1990', PlaylistConfig()) + assert deps.service.calls[0]['decade'] == 1990 + + def test_default_limit_is_100(self): + spec = get_registry().get('time_machine') + assert spec.default_config.limit == 100 + + def test_display_name_with_variant(self): + spec = get_registry().get('time_machine') + assert spec.display_name('1980s') == 'Time Machine — 1980s' + + +# ─── genre_playlist ───────────────────────────────────────────────── + + +class TestGenrePlaylist: + def test_registered(self): + spec = get_registry().get('genre_playlist') + assert spec is not None + assert spec.requires_variant is True + + def test_variant_resolver_normalizes_parent_keys(self): + spec = get_registry().get('genre_playlist') + variants = spec.variant_resolver(_deps()) + # 'Electronic/Dance' → 'electronic_dance' (slash → underscore + lowercase) + assert 'electronic_dance' in variants + assert 'hip_hop_rap' in variants + assert 'rock' in variants + + def test_normalized_variant_resolves_to_parent_key(self): + deps = _deps() + _gp_mod.generate(deps, 'electronic_dance', PlaylistConfig()) + # Service receives ORIGINAL parent key. + assert deps.service.calls[0]['genre'] == 'Electronic/Dance' + + def test_unknown_variant_passed_through_as_freeform(self): + # Service handles partial-matching for free-form keywords. + deps = _deps() + _gp_mod.generate(deps, 'shoegaze', PlaylistConfig()) + assert deps.service.calls[0]['genre'] == 'shoegaze' + + def test_empty_variant_raises(self): + deps = _deps() + with pytest.raises(ValueError, match='requires a variant'): + _gp_mod.generate(deps, '', PlaylistConfig()) + + def test_display_name(self): + spec = get_registry().get('genre_playlist') + assert spec.display_name('electronic_dance') == 'Genre — electronic_dance' diff --git a/tests/test_personalized_manager.py b/tests/test_personalized_manager.py new file mode 100644 index 00000000..6b12ca7a --- /dev/null +++ b/tests/test_personalized_manager.py @@ -0,0 +1,532 @@ +"""Boundary tests for the personalized-playlists foundation +(``core.personalized.types`` + ``core.personalized.specs`` + +``core.personalized.manager``). + +Pin every shape the storage layer + lifecycle has to handle so the +generators that arrive in subsequent commits can rely on a stable +contract: ensure_playlist auto-creates from default config, refresh +atomically replaces the snapshot + appends history, generator +exceptions don't lose the previous good snapshot, config patches +preserve unsent fields, recent_track_ids honors the day window, +list_playlists orders newest-first.""" + +from __future__ import annotations + +import os +import sqlite3 +import tempfile +from typing import Any, List +from unittest.mock import MagicMock + +import pytest + +from core.personalized.manager import PersonalizedPlaylistManager +from core.personalized.specs import PlaylistKindRegistry, PlaylistKindSpec +from core.personalized.types import PlaylistConfig, Track +from database.personalized_schema import ensure_personalized_schema + + +# ─── shared fixtures ───────────────────────────────────────────────── + + +class _FakeDB: + """Minimal MusicDatabase stand-in — gives the manager a real + sqlite connection so the manager exercises actual SQL.""" + + def __init__(self, path: str): + self.path = path + + def _get_connection(self): + conn = sqlite3.connect(self.path) + conn.row_factory = sqlite3.Row + return conn + + +@pytest.fixture +def db_path(tmp_path): + p = str(tmp_path / 'test.db') + conn = sqlite3.connect(p) + ensure_personalized_schema(conn) + conn.commit() + conn.close() + return p + + +@pytest.fixture +def db(db_path): + return _FakeDB(db_path) + + +@pytest.fixture +def registry(): + r = PlaylistKindRegistry() + return r + + +def _make_track(name='T1', artist='A1', sid='spot-1', source='spotify') -> Track: + return Track( + track_name=name, artist_name=artist, album_name='Album', + spotify_track_id=sid, source=source, + duration_ms=200000, popularity=50, + ) + + +# ─── PlaylistConfig ────────────────────────────────────────────────── + + +class TestPlaylistConfig: + def test_default_values(self): + c = PlaylistConfig() + assert c.limit == 50 + assert c.max_per_album == 2 + assert c.max_per_artist == 3 + assert c.popularity_min is None + assert c.popularity_max is None + assert c.exclude_recent_days == 0 + assert c.recency_days is None + assert c.seed is None + assert c.extra == {} + + def test_round_trip_through_json_dict(self): + c = PlaylistConfig( + limit=100, max_per_album=5, max_per_artist=10, + popularity_min=20, popularity_max=80, + exclude_recent_days=14, recency_days=180, + seed=42, extra={'selected_seasons': ['halloween', 'christmas']}, + ) + d = c.to_json_dict() + c2 = PlaylistConfig.from_json_dict(d) + assert c2 == c + + def test_from_json_dict_handles_none(self): + c = PlaylistConfig.from_json_dict(None) + assert c == PlaylistConfig() + + def test_from_json_dict_handles_non_dict(self): + c = PlaylistConfig.from_json_dict('garbage') # type: ignore + assert c == PlaylistConfig() + + def test_from_json_dict_missing_fields_use_defaults(self): + c = PlaylistConfig.from_json_dict({'limit': 75}) + assert c.limit == 75 + assert c.max_per_album == 2 # default + + def test_merged_overrides_only_named_fields(self): + base = PlaylistConfig(limit=50, popularity_min=20) + out = base.merged({'limit': 100}) + assert out.limit == 100 + assert out.popularity_min == 20 # untouched + + def test_merged_extra_dict_is_deep_merged(self): + base = PlaylistConfig(extra={'a': 1, 'b': 2}) + out = base.merged({'extra': {'b': 99, 'c': 3}}) + assert out.extra == {'a': 1, 'b': 99, 'c': 3} + + def test_merged_ignores_unknown_keys(self): + base = PlaylistConfig() + out = base.merged({'unknown_field': 'foo'}) + assert out == base + + +# ─── Track ──────────────────────────────────────────────────────────── + + +class TestTrack: + def test_from_dict_legacy_shape(self): + d = { + 'track_name': 'Song', 'artist_name': 'Band', + 'album_name': 'Album', 'spotify_track_id': 'spot-1', + 'duration_ms': 200000, 'popularity': 60, + '_artist_genres_raw': '["rock"]', # ignored extra + } + t = Track.from_dict(d) + assert t.track_name == 'Song' + assert t.spotify_track_id == 'spot-1' + assert t.duration_ms == 200000 + + def test_primary_id_prefers_spotify(self): + t = Track( + track_name='', artist_name='', + spotify_track_id='spot', itunes_track_id='itu', deezer_track_id='dee', + ) + assert t.primary_id() == 'spot' + + def test_primary_id_falls_back_through_sources(self): + t = Track(track_name='', artist_name='', itunes_track_id='itu') + assert t.primary_id() == 'itu' + t2 = Track(track_name='', artist_name='', deezer_track_id='dee') + assert t2.primary_id() == 'dee' + + def test_primary_id_none_when_no_sources(self): + t = Track(track_name='', artist_name='') + assert t.primary_id() is None + + +# ─── PlaylistKindRegistry ──────────────────────────────────────────── + + +class TestRegistry: + def test_register_and_get(self, registry): + spec = PlaylistKindSpec( + kind='hidden_gems', name_template='Hidden Gems', + description='', default_config=PlaylistConfig(), + generator=lambda *a, **k: [], + ) + registry.register(spec) + assert registry.get('hidden_gems') is spec + assert registry.get('nonexistent') is None + + def test_duplicate_registration_raises(self, registry): + spec = PlaylistKindSpec( + kind='x', name_template='X', description='', + default_config=PlaylistConfig(), generator=lambda *a, **k: [], + ) + registry.register(spec) + with pytest.raises(ValueError, match='already registered'): + registry.register(spec) + + def test_display_name_singleton(self): + spec = PlaylistKindSpec( + kind='x', name_template='Hidden Gems', description='', + default_config=PlaylistConfig(), generator=lambda *a, **k: [], + ) + assert spec.display_name('') == 'Hidden Gems' + + def test_display_name_with_variant(self): + spec = PlaylistKindSpec( + kind='x', name_template='Time Machine — {variant}', + description='', default_config=PlaylistConfig(), + generator=lambda *a, **k: [], + ) + assert spec.display_name('1980s') == 'Time Machine — 1980s' + + def test_kinds_listing(self, registry): + for k in ('a', 'b', 'c'): + registry.register(PlaylistKindSpec( + kind=k, name_template=k, description='', + default_config=PlaylistConfig(), generator=lambda *a, **k: [], + )) + assert set(registry.kinds()) == {'a', 'b', 'c'} + + +# ─── PersonalizedPlaylistManager ───────────────────────────────────── + + +def _register_simple_kind(registry, generator, kind='hidden_gems', requires_variant=False): + spec = PlaylistKindSpec( + kind=kind, name_template=kind.replace('_', ' ').title(), + description='', default_config=PlaylistConfig(limit=10), + generator=generator, requires_variant=requires_variant, + ) + registry.register(spec) + return spec + + +class TestEnsurePlaylist: + def test_creates_row_with_default_config(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.ensure_playlist('hidden_gems', '', 1) + assert record.id > 0 + assert record.kind == 'hidden_gems' + assert record.variant == '' + assert record.profile_id == 1 + assert record.config.limit == 10 # from default + assert record.track_count == 0 + assert record.last_generated_at is None + + def test_returns_same_row_on_second_call(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.ensure_playlist('hidden_gems', '', 1) + r2 = mgr.ensure_playlist('hidden_gems', '', 1) + assert r1.id == r2.id + + def test_variant_creates_separate_row(self, db, registry): + _register_simple_kind( + registry, lambda *a, **k: [], kind='time_machine', requires_variant=True, + ) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.ensure_playlist('time_machine', '1980s', 1) + r2 = mgr.ensure_playlist('time_machine', '1990s', 1) + assert r1.id != r2.id + + def test_unknown_kind_raises(self, db, registry): + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + with pytest.raises(ValueError, match='Unknown playlist kind'): + mgr.ensure_playlist('does_not_exist', '', 1) + + def test_required_variant_missing_raises(self, db, registry): + _register_simple_kind( + registry, lambda *a, **k: [], kind='time_machine', requires_variant=True, + ) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + with pytest.raises(ValueError, match='requires a variant'): + mgr.ensure_playlist('time_machine', '', 1) + + +class TestRefreshPlaylist: + def test_refresh_persists_tracks(self, db, registry): + tracks = [_make_track('S1', 'A1', 'sp1'), _make_track('S2', 'A1', 'sp2')] + _register_simple_kind(registry, lambda deps, variant, config: tracks) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.refresh_playlist('hidden_gems', '', 1) + assert record.track_count == 2 + assert record.last_generated_at is not None + assert record.last_generation_error is None + + persisted = mgr.get_playlist_tracks(record.id) + assert len(persisted) == 2 + assert persisted[0].track_name == 'S1' + assert persisted[1].track_name == 'S2' + + def test_refresh_replaces_previous_snapshot_atomically(self, db, registry): + run = {'tracks': [_make_track('first')]} + + def gen(deps, variant, config): + return run['tracks'] + + _register_simple_kind(registry, gen) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r1.track_count == 1 + + run['tracks'] = [_make_track('A'), _make_track('B'), _make_track('C')] + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r2.id == r1.id + assert r2.track_count == 3 + + persisted = mgr.get_playlist_tracks(r2.id) + assert [t.track_name for t in persisted] == ['A', 'B', 'C'] + + def test_generator_exception_preserves_previous_snapshot(self, db, registry): + run = {'mode': 'success'} + + def gen(deps, variant, config): + if run['mode'] == 'fail': + raise RuntimeError('generator boom') + return [_make_track('first')] + + _register_simple_kind(registry, gen) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r1.track_count == 1 + + run['mode'] = 'fail' + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + # Previous snapshot preserved. + assert r2.track_count == 1 + # Error stamped on row. + assert r2.last_generation_error is not None + assert 'generator boom' in r2.last_generation_error + # Tracks still queryable. + persisted = mgr.get_playlist_tracks(r2.id) + assert len(persisted) == 1 + + def test_config_overrides_passed_to_generator(self, db, registry): + captured = {} + + def gen(deps, variant, config): + captured['limit'] = config.limit + return [] + + _register_simple_kind(registry, gen) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1, config_overrides={'limit': 200}) + assert captured['limit'] == 200 + + def test_refresh_records_source_from_first_track(self, db, registry): + tracks = [_make_track(source='spotify'), _make_track(source='deezer')] + _register_simple_kind(registry, lambda *a, **k: tracks) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.refresh_playlist('hidden_gems', '', 1) + assert record.last_generation_source == 'spotify' + + def test_track_data_json_round_trips(self, db, registry): + nested = {'id': 'spot-1', 'name': 'Foo', 'artists': [{'name': 'Bar'}]} + track = Track( + track_name='Foo', artist_name='Bar', + spotify_track_id='spot-1', track_data_json=nested, + ) + _register_simple_kind(registry, lambda *a, **k: [track]) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.refresh_playlist('hidden_gems', '', 1) + persisted = mgr.get_playlist_tracks(record.id) + assert persisted[0].track_data_json == nested + + +class TestUpdateConfig: + def test_patch_merges_with_stored(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + record = mgr.update_config('hidden_gems', '', 1, {'limit': 75}) + assert record.config.limit == 75 + # Other fields kept. + assert record.config.max_per_album == 2 + + def test_patch_extra_dict_deep_merges(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.update_config('hidden_gems', '', 1, {'extra': {'a': 1}}) + record = mgr.update_config('hidden_gems', '', 1, {'extra': {'b': 2}}) + assert record.config.extra == {'a': 1, 'b': 2} + + +class TestListPlaylists: + def test_lists_all_playlists_for_profile(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [], kind='hidden_gems') + _register_simple_kind(registry, lambda *a, **k: [], kind='popular_picks') + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('popular_picks', '', 1) + records = mgr.list_playlists(1) + kinds = {r.kind for r in records} + assert kinds == {'hidden_gems', 'popular_picks'} + + def test_does_not_list_other_profiles(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('hidden_gems', '', 2) + assert len(mgr.list_playlists(1)) == 1 + assert len(mgr.list_playlists(2)) == 1 + + +class TestStalenessFilter: + """`config.exclude_recent_days > 0` drops tracks served by this + kind for this profile in the last N days.""" + + def test_zero_days_means_no_filter(self, db, registry): + # Default config has exclude_recent_days=0; everything passes. + tracks = [_make_track(sid='spot-1'), _make_track(sid='spot-2')] + run = {'tracks': tracks} + _register_simple_kind(registry, lambda *a, **k: run['tracks']) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + # Refresh again with same tracks — no filter, all should persist. + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r2.track_count == 2 + + def test_positive_days_filters_recently_served(self, db, registry): + run = {'tracks': [_make_track(sid='spot-1'), _make_track(sid='spot-2')]} + _register_simple_kind(registry, lambda *a, **k: run['tracks']) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r1.track_count == 2 + # Update config to exclude tracks served in last 7 days. + mgr.update_config('hidden_gems', '', 1, {'exclude_recent_days': 7}) + # Same generator output now → all tracks just got served, all filtered out. + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r2.track_count == 0 + + def test_filter_preserves_non_recent_tracks(self, db, registry): + run = {'tracks': [_make_track(sid='spot-1')]} + _register_simple_kind(registry, lambda *a, **k: run['tracks']) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + mgr.update_config('hidden_gems', '', 1, {'exclude_recent_days': 7}) + # New generator output with a NEW id — should pass. + run['tracks'] = [_make_track(sid='spot-1'), _make_track(sid='spot-NEW')] + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + # spot-1 was just served, dropped. spot-NEW is fresh, kept. + assert r2.track_count == 1 + persisted = mgr.get_playlist_tracks(r2.id) + assert persisted[0].spotify_track_id == 'spot-NEW' + + def test_tracks_without_primary_id_pass_through(self, db, registry): + # Track with no source IDs — primary_id() is None — staleness + # filter has nothing to dedupe on, so the track passes. + track_no_id = Track(track_name='X', artist_name='Y', source='spotify') + run = {'tracks': [track_no_id]} + _register_simple_kind(registry, lambda *a, **k: run['tracks']) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1) + mgr.update_config('hidden_gems', '', 1, {'exclude_recent_days': 7}) + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + # Track is kept because there's no id to match against history. + assert r2.track_count == 1 + + +class TestStaleFlag: + """`is_stale` flips when upstream data changes; refresh clears it.""" + + def test_default_is_false(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.ensure_playlist('hidden_gems', '', 1) + assert record.is_stale is False + + def test_mark_kinds_stale_flips_flag(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [], kind='hidden_gems') + _register_simple_kind(registry, lambda *a, **k: [], kind='discovery_shuffle') + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('discovery_shuffle', '', 1) + n = mgr.mark_kinds_stale(['hidden_gems', 'discovery_shuffle']) + assert n == 2 + assert mgr.ensure_playlist('hidden_gems', '', 1).is_stale is True + assert mgr.ensure_playlist('discovery_shuffle', '', 1).is_stale is True + + def test_mark_kinds_stale_only_matching_kinds(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [], kind='hidden_gems') + _register_simple_kind(registry, lambda *a, **k: [], kind='popular_picks') + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('popular_picks', '', 1) + mgr.mark_kinds_stale(['hidden_gems']) + assert mgr.ensure_playlist('hidden_gems', '', 1).is_stale is True + assert mgr.ensure_playlist('popular_picks', '', 1).is_stale is False + + def test_mark_kinds_stale_scopes_to_profile(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('hidden_gems', '', 2) + mgr.mark_kinds_stale(['hidden_gems'], profile_id=1) + assert mgr.ensure_playlist('hidden_gems', '', 1).is_stale is True + assert mgr.ensure_playlist('hidden_gems', '', 2).is_stale is False + + def test_refresh_clears_stale_flag(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [_make_track()]) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.mark_kinds_stale(['hidden_gems']) + assert mgr.ensure_playlist('hidden_gems', '', 1).is_stale is True + record = mgr.refresh_playlist('hidden_gems', '', 1) + assert record.is_stale is False + + def test_mark_kinds_stale_empty_list_noop(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + n = mgr.mark_kinds_stale([]) + assert n == 0 + + +class TestStalenessHistory: + def test_recent_track_ids_returns_zero_when_days_zero(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [_make_track(sid='spot-1')]) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1) + assert mgr.recent_track_ids(1, 'hidden_gems', 0) == [] + + def test_recent_track_ids_after_refresh(self, db, registry): + _register_simple_kind( + registry, + lambda *a, **k: [_make_track(sid='spot-1'), _make_track(sid='spot-2')], + ) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1) + recent = mgr.recent_track_ids(1, 'hidden_gems', 7) + assert set(recent) == {'spot-1', 'spot-2'} + + def test_recent_track_ids_scoped_to_kind(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [_make_track(sid='gem-1')], kind='hidden_gems') + _register_simple_kind(registry, lambda *a, **k: [_make_track(sid='pop-1')], kind='popular_picks') + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1) + mgr.refresh_playlist('popular_picks', '', 1) + assert mgr.recent_track_ids(1, 'hidden_gems', 7) == ['gem-1'] + assert mgr.recent_track_ids(1, 'popular_picks', 7) == ['pop-1'] diff --git a/tests/test_prowlarr_client.py b/tests/test_prowlarr_client.py new file mode 100644 index 00000000..eb3fe6c0 --- /dev/null +++ b/tests/test_prowlarr_client.py @@ -0,0 +1,223 @@ +"""Tests for ``core/prowlarr_client.py``. + +Pins the parse + dispatch behavior so a future Prowlarr API tweak +that drops a field doesn't silently lose data, and the search +endpoint keeps building the repeated-key query Prowlarr expects. +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import patch, MagicMock + +import pytest + +from core.prowlarr_client import ( + DEFAULT_MUSIC_CATEGORIES, + ProwlarrClient, + ProwlarrIndexer, + ProwlarrSearchResult, +) + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def _client_with_config(url="http://prowlarr:9696", api_key="secret"): + """Build a client whose ``_load_config`` already ran with the + given URL + key, sidestepping the real config_manager.""" + client = ProwlarrClient.__new__(ProwlarrClient) + client._url = url.rstrip('/') + client._api_key = api_key + return client + + +# --------------------------------------------------------------------------- +# Pure parsers +# --------------------------------------------------------------------------- + + +def test_parse_indexer_extracts_core_fields() -> None: + client = _client_with_config() + entry = { + 'id': 7, + 'name': 'Public Tracker', + 'protocol': 'torrent', + 'enable': True, + 'privacy': 'public', + 'capabilities': { + 'categories': [ + {'id': 3000, 'name': 'Audio'}, + {'id': 3040, 'name': 'Audio/Lossless'}, + ], + }, + } + indexer = client._parse_indexer(entry) + assert indexer == ProwlarrIndexer( + id=7, + name='Public Tracker', + protocol='torrent', + enable=True, + privacy='public', + categories=[3000, 3040], + capabilities=entry['capabilities'], + ) + + +def test_parse_indexer_tolerates_missing_capabilities() -> None: + """Some indexers (the ones in error state) come back with no + ``capabilities`` block — must not crash.""" + client = _client_with_config() + indexer = client._parse_indexer({'id': 1, 'name': 'X', 'protocol': 'usenet'}) + assert indexer.id == 1 + assert indexer.protocol == 'usenet' + assert indexer.categories == [] + + +def test_parse_result_extracts_torrent_fields() -> None: + client = _client_with_config() + entry = { + 'guid': 'guid-1', + 'title': 'Some Album FLAC', + 'indexerId': 3, + 'indexer': 'Tracker', + 'protocol': 'torrent', + 'downloadUrl': 'https://example.com/x.torrent', + 'magnetUrl': 'magnet:?xt=urn:btih:abc', + 'infoUrl': 'https://example.com/details/1', + 'size': 524288000, + 'seeders': 12, + 'leechers': 3, + 'grabs': 100, + 'publishDate': '2026-05-10T00:00:00Z', + 'categories': [{'id': 3040, 'name': 'Audio/Lossless'}], + } + result = client._parse_result(entry) + assert result.title == 'Some Album FLAC' + assert result.indexer_id == 3 + assert result.download_url == 'https://example.com/x.torrent' + assert result.magnet_uri == 'magnet:?xt=urn:btih:abc' + assert result.size == 524288000 + assert result.seeders == 12 + assert result.categories == [3040] + + +def test_parse_result_accepts_int_categories() -> None: + """Some indexers return categories as bare ints instead of + ``{id, name}`` dicts. Both forms must work.""" + client = _client_with_config() + result = client._parse_result({'title': 'X', 'categories': [3000, 3010]}) + assert result.categories == [3000, 3010] + + +def test_parse_result_skips_garbage_category_entries() -> None: + client = _client_with_config() + result = client._parse_result({'title': 'X', 'categories': [{'name': 'no-id'}, 'string', None]}) + assert result.categories == [] + + +# --------------------------------------------------------------------------- +# Configured-state predicates +# --------------------------------------------------------------------------- + + +def test_is_configured_requires_both_fields() -> None: + assert _client_with_config('http://x', '').is_configured() is False + assert _client_with_config('', 'key').is_configured() is False + assert _client_with_config('http://x', 'key').is_configured() is True + + +def test_check_connection_returns_false_when_not_configured() -> None: + client = _client_with_config('', '') + assert _run(client.check_connection()) is False + + +# --------------------------------------------------------------------------- +# HTTP plumbing +# --------------------------------------------------------------------------- + + +def _mock_response(status_code: int, json_body): + resp = MagicMock() + resp.ok = 200 <= status_code < 400 + resp.status_code = status_code + resp.json.return_value = json_body + return resp + + +def test_search_passes_repeated_categories_and_indexer_ids() -> None: + """Prowlarr's search endpoint expects repeated query keys — + ``categories=3000&categories=3010&indexerIds=1``. ``requests`` + serializes a list of tuples into that exact form, so we assert + the params are passed as a list-of-tuples (not a dict).""" + client = _client_with_config() + captured_params = {} + + def fake_get(url, headers=None, params=None, timeout=None): + captured_params['url'] = url + captured_params['params'] = params + return _mock_response(200, []) + + with patch('core.prowlarr_client.http_requests.get', side_effect=fake_get): + _run(client.search('the query', categories=[3000, 3010], indexer_ids=[1, 5])) + + assert captured_params['url'] == 'http://prowlarr:9696/api/v1/search' + params = captured_params['params'] + # Convert to a frozenset of pairs for order-independent comparison + pair_set = set(params) + assert ('query', 'the query') in pair_set + assert ('type', 'search') in pair_set + assert ('categories', 3000) in pair_set + assert ('categories', 3010) in pair_set + assert ('indexerIds', 1) in pair_set + assert ('indexerIds', 5) in pair_set + + +def test_search_returns_empty_on_blank_query() -> None: + client = _client_with_config() + # No HTTP mock — call must short-circuit without touching the network. + results = _run(client.search('')) + assert results == [] + results = _run(client.search(' ')) + assert results == [] + + +def test_search_parses_response_list() -> None: + client = _client_with_config() + with patch('core.prowlarr_client.http_requests.get', + return_value=_mock_response(200, [ + {'guid': 'a', 'title': 'Album A', 'protocol': 'torrent'}, + {'guid': 'b', 'title': 'Album B', 'protocol': 'usenet'}, + ])): + results = _run(client.search('q')) + assert [r.title for r in results] == ['Album A', 'Album B'] + assert [r.protocol for r in results] == ['torrent', 'usenet'] + + +def test_check_connection_hits_system_status() -> None: + client = _client_with_config() + with patch('core.prowlarr_client.http_requests.get', + return_value=_mock_response(200, {'version': '1.13.0'})) as mock_get: + ok = _run(client.check_connection()) + assert ok is True + called_url = mock_get.call_args.args[0] + assert called_url == 'http://prowlarr:9696/api/v1/system/status' + assert mock_get.call_args.kwargs['headers']['X-Api-Key'] == 'secret' + + +def test_check_connection_returns_false_on_http_error() -> None: + client = _client_with_config() + with patch('core.prowlarr_client.http_requests.get', + return_value=_mock_response(401, {'error': 'unauthorized'})): + ok = _run(client.check_connection()) + assert ok is False + + +def test_default_music_categories_match_newznab_tree() -> None: + """The Newznab Music category IDs are a stable convention across + Prowlarr / Jackett / every indexer. Pin the defaults so a typo + here doesn't silently broaden / narrow what SoulSync queries.""" + assert 3000 in DEFAULT_MUSIC_CATEGORIES # Audio (parent) + assert 3010 in DEFAULT_MUSIC_CATEGORIES # MP3 + assert 3040 in DEFAULT_MUSIC_CATEGORIES # Lossless diff --git a/tests/test_qobuz_playlists.py b/tests/test_qobuz_playlists.py new file mode 100644 index 00000000..828a9f97 --- /dev/null +++ b/tests/test_qobuz_playlists.py @@ -0,0 +1,456 @@ +"""Unit tests for QobuzClient playlist + favorites methods. + +Covers the Sync-page parity added for github issue #677: +- `get_user_playlists` paginates + normalizes the playlist list +- `get_playlist` paginates the tracklist + normalizes track shape +- `get_playlist` recognizes the virtual `qobuz-favorites` ID and + dispatches to `get_user_favorite_tracks` (same pattern as Tidal's + COLLECTION_PLAYLIST_ID) +- `get_user_favorite_tracks_count` reads the cheap count-only path +""" + +from __future__ import annotations + +import sys +import types +from typing import Any, Dict, List + +import pytest + + +@pytest.fixture +def qobuz_client_module(): + """Import core.qobuz_client with config_manager stubbed to a mutable + in-memory dict. Snapshots and restores sys.modules entries on + teardown so downstream tests still see the real config. + """ + config_state: Dict[str, Any] = {} + + class _StubConfigManager: + def get(self, key, default=None): + cur: Any = config_state + for part in key.split('.'): + if isinstance(cur, dict) and part in cur: + cur = cur[part] + else: + return default + return cur + + def set(self, key, value): + cur: Any = config_state + parts = key.split('.') + for part in parts[:-1]: + cur = cur.setdefault(part, {}) + cur[parts[-1]] = value + + original_modules = { + name: sys.modules.get(name) + for name in ('config', 'config.settings', 'core.qobuz_client') + } + + if 'config' not in sys.modules: + sys.modules['config'] = types.ModuleType('config') + settings_mod = types.ModuleType('config.settings') + settings_mod.config_manager = _StubConfigManager() + sys.modules['config.settings'] = settings_mod + + sys.modules.pop('core.qobuz_client', None) + try: + import core.qobuz_client as qobuz_client_module + yield qobuz_client_module, config_state + finally: + for name, original in original_modules.items(): + if original is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = original + + +@pytest.fixture +def authed_client(qobuz_client_module): + """A QobuzClient with stub credentials so is_authenticated() returns True.""" + module, config = qobuz_client_module + config['qobuz'] = { + 'session': { + 'app_id': 'APP-1', + 'app_secret': 'SECRET-1', + 'user_auth_token': 'TOKEN-1', + } + } + client = module.QobuzClient() + client.reload_credentials() + assert client.is_authenticated() is True + return client + + +def _install_api_responder(client, responder): + """Replace `_api_request` with a deterministic responder for the test.""" + client._api_request = responder # type: ignore[method-assign] + + +# --------------------------------------------------------------------------- +# get_user_playlists — pagination + normalization +# --------------------------------------------------------------------------- + + +def test_get_user_playlists_returns_normalized_metadata(authed_client): + calls: List[Dict[str, Any]] = [] + + def responder(endpoint, params=None): + calls.append({'endpoint': endpoint, 'params': params}) + return { + 'playlists': { + 'items': [ + { + 'id': 1001, + 'name': 'My Mix', + 'description': 'on repeat', + 'is_public': True, + 'tracks_count': 12, + 'images': ['https://qobuz.example/cover.jpg'], + }, + ], + 'total': 1, + } + } + + _install_api_responder(authed_client, responder) + playlists = authed_client.get_user_playlists() + + assert calls == [{ + 'endpoint': 'playlist/getUserPlaylists', + 'params': {'limit': 100, 'offset': 0}, + }] + assert playlists == [{ + 'id': '1001', + 'name': 'My Mix', + 'description': 'on repeat', + 'public': True, + 'track_count': 12, + 'image_url': 'https://qobuz.example/cover.jpg', + 'external_urls': {'qobuz': 'https://play.qobuz.com/playlist/1001'}, + }] + + +def test_get_user_playlists_paginates_until_total_reached(authed_client): + # Two pages of 100 each, third page returns empty to verify the loop + # terminates on `total` rather than waiting for an empty page. + page_one = [{'id': i, 'name': f'P{i}', 'tracks_count': 0} for i in range(100)] + page_two = [{'id': 100 + i, 'name': f'P{100 + i}', 'tracks_count': 0} for i in range(50)] + calls: List[int] = [] + + def responder(endpoint, params=None): + calls.append(params['offset']) + if params['offset'] == 0: + return {'playlists': {'items': page_one, 'total': 150}} + if params['offset'] == 100: + return {'playlists': {'items': page_two, 'total': 150}} + return {'playlists': {'items': [], 'total': 150}} + + _install_api_responder(authed_client, responder) + playlists = authed_client.get_user_playlists() + + assert len(playlists) == 150 + assert calls == [0, 100] # No third request needed + + +def test_get_user_playlists_returns_empty_when_unauthenticated(qobuz_client_module): + module, _ = qobuz_client_module + client = module.QobuzClient() # no credentials configured + assert client.is_authenticated() is False + + def responder(endpoint, params=None): + raise AssertionError('should not hit the API when unauthenticated') + + _install_api_responder(client, responder) + assert client.get_user_playlists() == [] + + +# --------------------------------------------------------------------------- +# get_playlist — track pagination + normalization +# --------------------------------------------------------------------------- + + +def test_get_playlist_normalizes_tracks(authed_client): + def responder(endpoint, params=None): + assert endpoint == 'playlist/get' + return { + 'id': 2002, + 'name': 'Deep Cuts', + 'description': '', + 'is_public': False, + 'tracks_count': 1, + 'images': ['https://qobuz.example/dc.jpg'], + 'tracks': { + 'items': [ + { + 'id': 555, + 'title': 'Forgotten Track', + 'duration': 240, + 'parental_warning': True, + 'performer': {'name': 'Some Artist'}, + 'album': { + 'title': 'Some Album', + 'image': {'large': 'https://qobuz.example/art.jpg'}, + }, + }, + ], + 'total': 1, + }, + } + + _install_api_responder(authed_client, responder) + playlist = authed_client.get_playlist('2002') + + assert playlist is not None + assert playlist['id'] == '2002' + assert playlist['name'] == 'Deep Cuts' + assert playlist['track_count'] == 1 + assert playlist['tracks'] == [{ + 'id': '555', + 'name': 'Forgotten Track', + 'artists': ['Some Artist'], + 'album': 'Some Album', + 'duration_ms': 240_000, + 'image_url': 'https://qobuz.example/art.jpg', + 'external_urls': {'qobuz': 'https://play.qobuz.com/track/555'}, + 'explicit': True, + }] + + +def test_get_playlist_routes_favorites_virtual_id(authed_client): + """The virtual `qobuz-favorites` ID must dispatch to the favorites + endpoint rather than the playlist/get endpoint — mirrors Tidal's + COLLECTION_PLAYLIST_ID pattern.""" + seen_endpoints: List[str] = [] + + def responder(endpoint, params=None): + seen_endpoints.append(endpoint) + # favorite/getUserFavorites is the only endpoint that should fire + return { + 'tracks': { + 'items': [ + { + 'id': 777, + 'title': 'Liked Song', + 'duration': 180, + 'performer': {'name': 'Loved Artist'}, + 'album': {'title': 'Heart Album', 'image': {'large': 'https://q.example/h.jpg'}}, + }, + ], + 'total': 1, + } + } + + _install_api_responder(authed_client, responder) + playlist = authed_client.get_playlist(authed_client.QOBUZ_FAVORITES_ID) + + assert playlist is not None + assert playlist['id'] == authed_client.QOBUZ_FAVORITES_ID + assert playlist['name'] == authed_client.QOBUZ_FAVORITES_NAME + assert playlist['track_count'] == 1 + assert playlist['tracks'][0]['name'] == 'Liked Song' + # Only the favorites endpoint should have been hit — no playlist/get. + assert seen_endpoints == ['favorite/getUserFavorites'] + + +def test_get_playlist_paginates_track_list(authed_client): + page_one_tracks = [ + {'id': i, 'title': f'T{i}', 'duration': 100, 'performer': {'name': 'A'}, 'album': {'title': 'Alb', 'image': {}}} + for i in range(100) + ] + page_two_tracks = [ + {'id': 100 + i, 'title': f'T{100 + i}', 'duration': 100, 'performer': {'name': 'A'}, 'album': {'title': 'Alb', 'image': {}}} + for i in range(25) + ] + offsets: List[int] = [] + + def responder(endpoint, params=None): + offsets.append(params['offset']) + if params['offset'] == 0: + return { + 'id': 'X', 'name': 'Long', 'description': '', 'is_public': False, + 'tracks_count': 125, 'images': [], + 'tracks': {'items': page_one_tracks, 'total': 125}, + } + if params['offset'] == 100: + return { + 'id': 'X', 'name': 'Long', 'description': '', 'is_public': False, + 'tracks_count': 125, 'images': [], + 'tracks': {'items': page_two_tracks, 'total': 125}, + } + return {'tracks': {'items': [], 'total': 125}} + + _install_api_responder(authed_client, responder) + playlist = authed_client.get_playlist('X') + + assert playlist is not None + assert len(playlist['tracks']) == 125 + assert playlist['track_count'] == 125 + assert offsets == [0, 100] + + +def test_get_playlist_returns_none_when_unauthenticated(qobuz_client_module): + module, _ = qobuz_client_module + client = module.QobuzClient() + assert client.get_playlist('whatever') is None + + +# --------------------------------------------------------------------------- +# get_user_favorite_tracks + get_user_favorite_tracks_count +# --------------------------------------------------------------------------- + + +def test_get_user_favorite_tracks_paginates(authed_client): + def make_items(start, count): + return [ + {'id': start + i, 'title': f'F{start + i}', 'duration': 200, + 'performer': {'name': 'Fav Artist'}, + 'album': {'title': 'Fav Album', 'image': {}}} + for i in range(count) + ] + + offsets: List[int] = [] + + def responder(endpoint, params=None): + assert endpoint == 'favorite/getUserFavorites' + assert params['type'] == 'tracks' + offsets.append(params['offset']) + if params['offset'] == 0: + return {'tracks': {'items': make_items(0, 100), 'total': 130}} + if params['offset'] == 100: + return {'tracks': {'items': make_items(100, 30), 'total': 130}} + return {'tracks': {'items': [], 'total': 130}} + + _install_api_responder(authed_client, responder) + tracks = authed_client.get_user_favorite_tracks() + + assert len(tracks) == 130 + assert offsets == [0, 100] + assert tracks[0]['name'] == 'F0' + assert tracks[-1]['name'] == 'F129' + + +def test_get_user_favorite_tracks_fetches_all_by_default(authed_client): + def make_items(start, count): + return [ + {'id': start + i, 'title': f'F{start + i}', 'duration': 200, + 'performer': {'name': 'Fav Artist'}, + 'album': {'title': 'Fav Album', 'image': {}}} + for i in range(count) + ] + + offsets: List[int] = [] + + def responder(endpoint, params=None): + assert endpoint == 'favorite/getUserFavorites' + offsets.append(params['offset']) + start = params['offset'] + remaining = max(0, 625 - start) + return {'tracks': {'items': make_items(start, min(params['limit'], remaining)), 'total': 625}} + + _install_api_responder(authed_client, responder) + tracks = authed_client.get_user_favorite_tracks() + + assert len(tracks) == 625 + assert offsets == [0, 100, 200, 300, 400, 500, 600] + assert tracks[-1]['name'] == 'F624' + + +def test_get_user_favorite_tracks_honors_explicit_limit(authed_client): + def make_items(start, count): + return [ + {'id': start + i, 'title': f'F{start + i}', 'duration': 200, + 'performer': {'name': 'Fav Artist'}, + 'album': {'title': 'Fav Album', 'image': {}}} + for i in range(count) + ] + + requests: List[Dict[str, Any]] = [] + + def responder(endpoint, params=None): + assert endpoint == 'favorite/getUserFavorites' + requests.append(dict(params)) + start = params['offset'] + return {'tracks': {'items': make_items(start, params['limit']), 'total': 625}} + + _install_api_responder(authed_client, responder) + tracks = authed_client.get_user_favorite_tracks(limit=150) + + assert len(tracks) == 150 + assert requests == [ + {'type': 'tracks', 'limit': 100, 'offset': 0}, + {'type': 'tracks', 'limit': 50, 'offset': 100}, + ] + assert tracks[-1]['name'] == 'F149' + + +def test_get_user_favorite_tracks_count_uses_cheap_call(authed_client): + captured: Dict[str, Any] = {} + + def responder(endpoint, params=None): + captured['endpoint'] = endpoint + captured['params'] = params + return {'tracks': {'items': [], 'total': 4242}} + + _install_api_responder(authed_client, responder) + count = authed_client.get_user_favorite_tracks_count() + + assert count == 4242 + # Single request with limit=1 — must not iterate the full list. + assert captured == { + 'endpoint': 'favorite/getUserFavorites', + 'params': {'type': 'tracks', 'limit': 1, 'offset': 0}, + } + + +def test_get_user_favorite_tracks_count_returns_zero_when_unauthenticated(qobuz_client_module): + module, _ = qobuz_client_module + client = module.QobuzClient() + assert client.get_user_favorite_tracks_count() == 0 + + +# --------------------------------------------------------------------------- +# Track normalization fallbacks — artist resolution chain +# --------------------------------------------------------------------------- + + +def test_track_normalization_falls_back_to_album_artist(authed_client): + """When `performer.name` is missing, album.artist.name should win + over the bare 'Unknown Artist' default.""" + def responder(endpoint, params=None): + return { + 'id': 'P', 'name': 'p', 'description': '', 'is_public': False, + 'tracks_count': 1, 'images': [], + 'tracks': { + 'items': [{ + 'id': 1, 'title': 'X', 'duration': 10, + 'album': { + 'title': 'A', + 'artist': {'name': 'Album Artist'}, + 'image': {'large': ''}, + }, + }], + 'total': 1, + } + } + + _install_api_responder(authed_client, responder) + playlist = authed_client.get_playlist('P') + assert playlist['tracks'][0]['artists'] == ['Album Artist'] + + +def test_track_normalization_uses_unknown_artist_when_all_sources_empty(authed_client): + def responder(endpoint, params=None): + return { + 'id': 'P', 'name': 'p', 'description': '', 'is_public': False, + 'tracks_count': 1, 'images': [], + 'tracks': { + 'items': [{'id': 1, 'title': 'X', 'duration': 10}], + 'total': 1, + } + } + + _install_api_responder(authed_client, responder) + playlist = authed_client.get_playlist('P') + assert playlist['tracks'][0]['artists'] == ['Unknown Artist'] diff --git a/tests/test_reorganize_tag_source.py b/tests/test_reorganize_tag_source.py new file mode 100644 index 00000000..9eee1ff5 --- /dev/null +++ b/tests/test_reorganize_tag_source.py @@ -0,0 +1,602 @@ +"""Boundary tests for ``core.library.reorganize_tag_source``. + +Pin every shape the embedded-tag → reorganize-context adapter has to +handle so future drift fails here instead of at runtime against a +real library: empty / missing essentials, multi-value vs single-string +artist tags, ID3-style ``"5/12"`` track-number values, year +normalization across date shapes, releasetype validation, multi-disc +parsing, defensive paths against bad input. + +The wrapper :func:`read_album_track_from_file` is tested against a +fake ``read_embedded_tags_fn`` so no real mutagen IO happens here. +""" + +from __future__ import annotations + +import sys +import types +from typing import Any, Dict + +import pytest + + +# ── stubs (other tests rely on these too — keep the shape consistent) ── +if 'utils.logging_config' not in sys.modules: + utils_mod = types.ModuleType('utils') + logging_mod = types.ModuleType('utils.logging_config') + logging_mod.get_logger = lambda name: type('L', (), { + 'debug': lambda *a, **k: None, + 'info': lambda *a, **k: None, + 'warning': lambda *a, **k: None, + 'error': lambda *a, **k: None, + })() + sys.modules['utils'] = utils_mod + sys.modules['utils.logging_config'] = logging_mod + + +from core.library.reorganize_tag_source import ( + extract_album_meta_from_tags, + extract_track_meta_from_tags, + read_album_track_from_file, + normalize_resolved_path, +) + + +# ─── extract_track_meta_from_tags ───────────────────────────────────── + + +class TestExtractTrackMeta: + def test_full_set_returns_canonical_shape(self): + out = extract_track_meta_from_tags({ + 'title': 'HUMBLE.', + 'artist': 'Kendrick Lamar', + 'tracknumber': '4', + 'discnumber': '1', + }) + assert out is not None + assert out['name'] == 'HUMBLE.' + assert out['title'] == 'HUMBLE.' + assert out['track_number'] == 4 + assert out['disc_number'] == 1 + assert out['artists'] == [{'name': 'Kendrick Lamar'}] + assert out['duration_ms'] == 0 + assert out['id'] == '' + assert out['uri'] == '' + + def test_missing_title_returns_none(self): + assert extract_track_meta_from_tags({'artist': 'foo'}) is None + assert extract_track_meta_from_tags({'title': '', 'artist': 'foo'}) is None + assert extract_track_meta_from_tags({'title': ' ', 'artist': 'foo'}) is None + + def test_missing_artist_returns_none(self): + assert extract_track_meta_from_tags({'title': 'Song'}) is None + assert extract_track_meta_from_tags({'title': 'Song', 'artist': ''}) is None + + def test_multi_value_artists_field_takes_precedence(self): + out = extract_track_meta_from_tags({ + 'title': 'Collab', + 'artist': 'Foo Bar', + 'artists': 'Foo, Bar, Baz', # multi-value tag joined by reader + }) + assert out is not None + assert out['artists'] == [{'name': 'Foo'}, {'name': 'Bar'}, {'name': 'Baz'}] + + def test_artist_string_split_on_known_separators(self): + for sep_input, expected in [ + ('Foo, Bar', ['Foo', 'Bar']), + ('Foo & Bar', ['Foo', 'Bar']), + ('Foo feat. Bar', ['Foo', 'Bar']), + ('Foo ft Bar', ['Foo', 'Bar']), + ('Foo featuring Bar', ['Foo', 'Bar']), + ('Foo / Bar', ['Foo', 'Bar']), + ('Foo; Bar', ['Foo', 'Bar']), + ('Foo x Bar', ['Foo', 'Bar']), + ('Foo with Bar', ['Foo', 'Bar']), + ]: + out = extract_track_meta_from_tags({'title': 't', 'artist': sep_input}) + assert out is not None + names = [a['name'] for a in out['artists']] + assert names == expected, f"failed for {sep_input!r}: got {names}" + + def test_artist_dedup_case_insensitive(self): + out = extract_track_meta_from_tags({ + 'title': 't', + 'artists': 'Foo, foo, FOO, Bar', + }) + assert out is not None + names = [a['name'] for a in out['artists']] + assert names == ['Foo', 'Bar'] + + def test_id3_track_total_shape(self): + # ID3 stores TRCK as "5/12" — caller must use the head only. + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': '5/12', + }) + assert out['track_number'] == 5 + + def test_disc_total_shape(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'discnumber': '2/2', + }) + assert out['disc_number'] == 2 + + def test_track_number_default_to_one(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + }) + assert out['track_number'] == 1 + assert out['disc_number'] == 1 + + def test_track_number_zero_or_negative_falls_back_to_one(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', 'tracknumber': '0', + }) + assert out['track_number'] == 1 # or-default of 0 → 1 + + def test_track_number_unparseable_falls_back_to_one(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': 'side-a-2', + }) + assert out['track_number'] == 1 + + def test_int_track_number(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': 5, + 'discnumber': 2, + }) + assert out['track_number'] == 5 + assert out['disc_number'] == 2 + + def test_float_track_number_truncated(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': 5.7, + }) + assert out['track_number'] == 5 + + def test_zero_padded_track_number(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': '03', + }) + assert out['track_number'] == 3 + + def test_non_dict_input(self): + assert extract_track_meta_from_tags(None) is None + assert extract_track_meta_from_tags([]) is None + assert extract_track_meta_from_tags('') is None + + def test_empty_dict(self): + assert extract_track_meta_from_tags({}) is None + + +# ─── extract_album_meta_from_tags ───────────────────────────────────── + + +class TestExtractAlbumMeta: + def test_full_set(self): + out = extract_album_meta_from_tags({ + 'album': 'DAMN.', + 'albumartist': 'Kendrick Lamar', + 'date': '2017-04-14', + 'totaltracks': '14', + 'releasetype': 'Album', + }) + assert out['name'] == 'DAMN.' + assert out['title'] == 'DAMN.' + assert out['album_artist'] == 'Kendrick Lamar' + assert out['release_date'] == '2017' + assert out['total_tracks'] == 14 + assert out['album_type'] == 'album' + assert out['image_url'] == '' + assert out['id'] == '' + + def test_year_normalization_from_full_date(self): + for date_input, expected_year in [ + ('2020-01-15', '2020'), + ('2020', '2020'), + ('2020-01', '2020'), + ('Jan 5, 2020', '2020'), + ('1999/12/31', '1999'), + ]: + out = extract_album_meta_from_tags({'album': 'a', 'date': date_input}) + assert out['release_date'] == expected_year, f"date={date_input!r}" + + def test_year_falls_back_to_year_field(self): + out = extract_album_meta_from_tags({'album': 'a', 'year': '2018'}) + assert out['release_date'] == '2018' + + def test_year_falls_back_to_originaldate(self): + out = extract_album_meta_from_tags({'album': 'a', 'originaldate': '2010'}) + assert out['release_date'] == '2010' + + def test_year_missing_returns_empty(self): + out = extract_album_meta_from_tags({'album': 'a'}) + assert out['release_date'] == '' + + def test_totaltracks_from_id3_shape(self): + # ID3 may store track_number as "5/12" — use the trailing 12. + out = extract_album_meta_from_tags({ + 'album': 'a', 'tracknumber': '5/12', + }) + assert out['total_tracks'] == 12 + + def test_totaltracks_explicit_field_wins(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'totaltracks': '14', 'tracknumber': '5/12', + }) + assert out['total_tracks'] == 14 + + def test_totaltracks_tracktotal_alias(self): + out = extract_album_meta_from_tags({'album': 'a', 'tracktotal': '8'}) + assert out['total_tracks'] == 8 + + def test_releasetype_canonical(self): + for input_val, expected in [ + ('album', 'album'), ('Album', 'album'), + ('single', 'single'), ('Single', 'single'), + ('ep', 'ep'), ('EP', 'ep'), + ('compilation', 'compilation'), + ('soundtrack', ''), # not in canonical set + ('mixtape', ''), + ('', ''), + ]: + out = extract_album_meta_from_tags({ + 'album': 'a', 'releasetype': input_val, + }) + assert out['album_type'] == expected, f"releasetype={input_val!r}" + + def test_total_discs_explicit_field(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'totaldiscs': '2', + }) + assert out['total_discs'] == 2 + + def test_total_discs_from_id3_disc_form(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'discnumber': '1/2', + }) + assert out['total_discs'] == 2 + + def test_total_discs_explicit_wins_over_disc_form(self): + # When both present, take the larger (defensive against drift). + out = extract_album_meta_from_tags({ + 'album': 'a', 'discnumber': '1/2', 'totaldiscs': '3', + }) + assert out['total_discs'] == 3 + + def test_total_discs_missing_zero(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'discnumber': '1', + }) + assert out['total_discs'] == 0 # caller defaults via max() with disc count + + def test_album_artist_underscore_alias(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'album_artist': 'Foo', + }) + assert out['album_artist'] == 'Foo' + + def test_missing_album_returns_empty_name(self): + out = extract_album_meta_from_tags({}) + assert out['name'] == '' + # All other fields should still be present (zero/empty), so the + # caller's downstream consumer doesn't KeyError. + assert 'release_date' in out + assert 'total_tracks' in out + + def test_non_dict_input_safe(self): + out = extract_album_meta_from_tags(None) # type: ignore + assert out['name'] == '' + + +# ─── read_album_track_from_file ─────────────────────────────────────── + + +class TestReadAlbumTrackFromFile: + def test_unavailable_result_returns_reason(self): + def fake_reader(_p): + return {'available': False, 'reason': 'No file.'} + + a, t, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert a is None and t is None + assert err == 'No file.' + + def test_unavailable_no_reason_falls_back(self): + def fake_reader(_p): + return {'available': False} + + _, _, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert 'Could not read embedded tags' in (err or '') + + def test_non_dict_result_safe(self): + def fake_reader(_p): + return None + + a, t, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert a is None and t is None + assert err + + def test_essentials_missing_track_returns_reason(self): + # Title missing → unmatched. + def fake_reader(_p): + return {'available': True, 'tags': {'artist': 'a', 'album': 'b'}} + + a, t, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert a is None and t is None + assert 'title' in (err or '').lower() + + def test_essentials_missing_album_returns_reason(self): + # Album missing → unmatched even if track meta extracted. + def fake_reader(_p): + return {'available': True, 'tags': {'title': 't', 'artist': 'a'}} + + a, t, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert a is None and t is None + assert 'album' in (err or '').lower() + + def test_full_extraction(self): + def fake_reader(_p): + return { + 'available': True, + 'duration': 234.5, + 'tags': { + 'title': 'HUMBLE.', + 'artist': 'Kendrick Lamar', + 'album': 'DAMN.', + 'albumartist': 'Kendrick Lamar', + 'tracknumber': '4/14', + 'discnumber': '1/1', + 'date': '2017-04-14', + 'releasetype': 'Album', + }, + } + + album, track, err = read_album_track_from_file( + '/fake.flac', read_embedded_tags_fn=fake_reader, + ) + assert err is None + assert track is not None and album is not None + assert track['name'] == 'HUMBLE.' + assert track['track_number'] == 4 + assert track['disc_number'] == 1 + assert track['duration_ms'] == 234500 + assert album['name'] == 'DAMN.' + assert album['release_date'] == '2017' + assert album['total_tracks'] == 14 + assert album['album_type'] == 'album' + + def test_duration_zero_when_missing(self): + def fake_reader(_p): + return { + 'available': True, + 'tags': { + 'title': 't', 'artist': 'a', 'album': 'b', + }, + } + + _, track, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert err is None + assert track['duration_ms'] == 0 + + def test_duration_unparseable_zero(self): + def fake_reader(_p): + return { + 'available': True, + 'duration': 'banana', + 'tags': {'title': 't', 'artist': 'a', 'album': 'b'}, + } + + _, track, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert err is None + assert track['duration_ms'] == 0 + + def test_empty_path(self): + a, t, err = read_album_track_from_file('') + assert a is None and t is None + assert err + + def test_non_string_path(self): + a, t, err = read_album_track_from_file(None) # type: ignore + assert a is None and t is None + assert err + + +# ─── normalize_resolved_path ────────────────────────────────────────── + + +class TestNormalizeResolvedPath: + def test_returns_path_when_exists(self, tmp_path): + f = tmp_path / 'x.flac' + f.write_bytes(b'') + assert normalize_resolved_path(str(f)) == str(f) + + def test_none_when_missing(self, tmp_path): + assert normalize_resolved_path(str(tmp_path / 'no.flac')) is None + + def test_empty_input_safe(self): + assert normalize_resolved_path('') is None + assert normalize_resolved_path(None) is None + + +# ─── plan_album_reorganize (tag-mode integration) ──────────────────── +# +# Pin the wiring between the planner branch and the tag-source helper +# so the additive-and-optional contract holds: API mode unchanged, +# tag mode produces matched plan items shaped like API mode (so +# downstream post-process treats them identically). + + +def _stub_metadata_service(monkeypatch): + """Inject a minimal `core.metadata_service` so `library_reorganize` + imports cleanly even in the test process where the real metadata + clients aren't wired.""" + if 'core' not in sys.modules: + sys.modules['core'] = types.ModuleType('core') + if 'core.metadata_service' in sys.modules: + return + fake = types.ModuleType('core.metadata_service') + fake.get_album_for_source = lambda *a, **k: {} + fake.get_album_tracks_for_source = lambda *a, **k: [] + fake.get_client_for_source = lambda *a, **k: None + fake.get_primary_source = lambda: 'deezer' + fake.get_source_priority = lambda primary=None: ['deezer', 'spotify', 'itunes'] + sys.modules['core.metadata_service'] = fake + + +class TestPlannerTagModeIntegration: + def test_tag_mode_planner_matches_every_track_with_good_tags(self, monkeypatch): + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + per_path = { + '/a/track1.flac': { + 'available': True, 'duration': 200, + 'tags': { + 'title': 'Song A', 'artist': 'Foo', 'album': 'AlbumX', + 'tracknumber': '1/3', 'discnumber': '1/1', + 'date': '2020', 'releasetype': 'Album', + }, + }, + '/a/track2.flac': { + 'available': True, 'duration': 230, + 'tags': { + 'title': 'Song B', 'artist': 'Foo', 'album': 'AlbumX', + 'tracknumber': '2/3', 'discnumber': '1/1', + 'date': '2020', 'releasetype': 'Album', + }, + }, + } + monkeypatch.setattr( + 'core.library.file_tags.read_embedded_tags', + lambda p: per_path.get(p, {'available': False, 'reason': 'missing'}), + ) + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'Foo', 'title': 'AlbumX'}, + tracks=[ + {'id': 't1', 'title': 'Song A', 'track_number': 1, 'file_path': '/a/track1.flac'}, + {'id': 't2', 'title': 'Song B', 'track_number': 2, 'file_path': '/a/track2.flac'}, + ], + metadata_source='tags', + resolve_file_path_fn=lambda p: p, + ) + assert plan['status'] == 'planned' + assert plan['source'] == 'tags' + assert plan['total_discs'] == 1 + assert len(plan['items']) == 2 + for it in plan['items']: + assert it['matched'] is True + assert it['api_track']['name'] in ('Song A', 'Song B') + assert it['api_album']['name'] == 'AlbumX' + assert it['api_album']['album_type'] == 'album' + + def test_tag_mode_partial_disc_uses_tagged_total_discs(self, monkeypatch): + # User has only disc 2 of a 2-disc album; tags say so. + # max_disc must reflect tagged total so path builder still + # routes into the multi-disc subfolder. + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + monkeypatch.setattr( + 'core.library.file_tags.read_embedded_tags', + lambda p: { + 'available': True, + 'tags': { + 'title': 'Song A', 'artist': 'Foo', 'album': 'Y', + 'tracknumber': '1/8', 'discnumber': '2/2', + 'totaldiscs': '2', + }, + }, + ) + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'Foo', 'title': 'Y'}, + tracks=[{'id': 't1', 'title': 'Song A', 'track_number': 1, 'file_path': '/a.flac'}], + metadata_source='tags', + resolve_file_path_fn=lambda p: p, + ) + assert plan['status'] == 'planned' + assert plan['total_discs'] == 2 + + def test_tag_mode_file_missing_unmatched_with_reason(self, monkeypatch): + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'Foo', 'title': 'X'}, + tracks=[{'id': 't1', 'title': 'Song', 'track_number': 1, 'file_path': '/missing.flac'}], + metadata_source='tags', + resolve_file_path_fn=lambda p: None, # always missing + ) + # All tracks unmatched → no_source_id status, source='tags'. + assert plan['status'] == 'no_source_id' + assert plan['source'] == 'tags' + assert plan['items'][0]['matched'] is False + assert 'no longer exists' in plan['items'][0]['reason'].lower() + + def test_tag_mode_some_match_some_unreadable(self, monkeypatch): + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + per_path = { + '/good.flac': { + 'available': True, + 'tags': {'title': 'Good', 'artist': 'A', 'album': 'X', 'tracknumber': '1/2'}, + }, + '/bad.flac': {'available': False, 'reason': 'unreadable'}, + } + monkeypatch.setattr( + 'core.library.file_tags.read_embedded_tags', + lambda p: per_path.get(p, {'available': False, 'reason': 'missing'}), + ) + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'A', 'title': 'X'}, + tracks=[ + {'id': 'g', 'title': 'Good', 'track_number': 1, 'file_path': '/good.flac'}, + {'id': 'b', 'title': 'Bad', 'track_number': 2, 'file_path': '/bad.flac'}, + ], + metadata_source='tags', + resolve_file_path_fn=lambda p: p, + ) + assert plan['status'] == 'planned' + assert plan['source'] == 'tags' + matched = [it for it in plan['items'] if it['matched']] + unmatched = [it for it in plan['items'] if not it['matched']] + assert len(matched) == 1 + assert len(unmatched) == 1 + assert unmatched[0]['reason'] == 'unreadable' + + def test_tag_mode_without_resolver_returns_no_source_id(self, monkeypatch): + # Defensive: caller forgot to pass resolve_file_path_fn. + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'A', 'title': 'X'}, + tracks=[{'id': 't1', 'title': 'Song', 'track_number': 1, 'file_path': '/a.flac'}], + metadata_source='tags', + resolve_file_path_fn=None, + ) + assert plan['status'] == 'no_source_id' + assert plan['items'][0]['matched'] is False + assert 'requires the file path resolver' in plan['items'][0]['reason'] + + def test_api_mode_unchanged_default(self, monkeypatch): + # Regression guard: omitting metadata_source preserves the API + # path — calls _resolve_source which calls our stubbed + # metadata_service. Should land in 'no_source_id' since stubs + # return empty. + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'Foo', 'title': 'Bar'}, + tracks=[{'id': 't1', 'title': 'Song', 'track_number': 1, 'file_path': '/a.flac'}], + ) + # No metadata_source param → defaults to 'api' → empty stubs + # produce no_source_id. + assert plan['status'] == 'no_source_id' + assert plan['source'] is None # never reached the tags branch diff --git a/tests/test_repair_worker_album_fill.py b/tests/test_repair_worker_album_fill.py index 57a44bcd..ee1bf28e 100644 --- a/tests/test_repair_worker_album_fill.py +++ b/tests/test_repair_worker_album_fill.py @@ -44,6 +44,167 @@ if "config.settings" not in sys.modules: from core.repair_worker import RepairWorker +def test_incomplete_album_auto_fill_skips_source_artist_mismatch(tmp_path): + """Album Completeness must never fill a target album with another artist.""" + existing_path = tmp_path / "Gut" / "Light Years" / "01 - Wound Fuck.flac" + existing_path.parent.mkdir(parents=True) + existing_path.write_bytes(b"existing") + + candidate_path = tmp_path / "Jamiroquai" / "Light Years.flac" + candidate_path.parent.mkdir(parents=True) + candidate_path.write_bytes(b"candidate") + + class _FakeDB: + def __init__(self): + self.search_calls = [] + self.wishlist = [] + + def get_tracks_by_album(self, album_id): + if album_id == "target-album": + return [ + SimpleNamespace( + id="target-existing-1", + album_id="target-album", + artist_id="gut", + title="Wound Fuck", + track_number=1, + file_path=str(existing_path), + bitrate=9999, + ), + ] + return [] + + def search_tracks(self, title="", artist="", limit=50, server_source=None): + self.search_calls.append((title, artist, limit, server_source)) + return [ + SimpleNamespace( + id="candidate-1", + album_id="single-album", + artist_id="jamiroquai", + artist_name="Jamiroquai", + title="Light Years", + track_number=1, + file_path=str(candidate_path), + bitrate=9999, + ), + ] + + def add_to_wishlist(self, *args, **kwargs): + self.wishlist.append((args, kwargs)) + + worker = RepairWorker.__new__(RepairWorker) + worker.db = _FakeDB() + worker.transfer_folder = str(tmp_path) + worker._config_manager = None + + result = worker._fix_incomplete_album( + "album", + "target-album", + None, + { + "album_id": "target-album", + "album_title": "Light Years", + "artist": "Gut", + "missing_tracks": [ + { + "name": "Light Years", + "track_number": 2, + "disc_number": 1, + "source": "spotify", + "source_track_id": "sp-light-years", + "artists": ["Jamiroquai"], + }, + ], + }, + ) + + assert result["success"] is False + assert result["fixed"] == 0 + assert result["skipped"] == 1 + assert result["details"][0]["reason"] == "source artist does not match target album artist" + assert worker.db.search_calls == [] + assert worker.db.wishlist == [] + + +def test_incomplete_album_auto_fill_rejects_wrong_artist_candidate(tmp_path): + """Exact title is not enough when the candidate artist differs.""" + existing_path = tmp_path / "album" / "01 - Existing.flac" + existing_path.parent.mkdir(parents=True) + existing_path.write_bytes(b"existing") + candidate_path = tmp_path / "wrong" / "02 - Light Years.flac" + candidate_path.parent.mkdir(parents=True) + candidate_path.write_bytes(b"wrong") + + class _FakeDB: + def __init__(self): + self.wishlist = [] + + def get_tracks_by_album(self, album_id): + if album_id == "target-album": + return [ + SimpleNamespace( + id="target-existing-1", + album_id="target-album", + artist_id="jamiroquai", + title="Existing", + track_number=1, + file_path=str(existing_path), + bitrate=9999, + ), + ] + return [] + + def search_tracks(self, title="", artist="", limit=50, server_source=None): + return [ + SimpleNamespace( + id="candidate-1", + album_id="wrong-album", + artist_id="gut", + artist_name="Gut", + title="Light Years", + track_number=1, + file_path=str(candidate_path), + bitrate=9999, + ), + ] + + def add_to_wishlist(self, *args, **kwargs): + self.wishlist.append((args, kwargs)) + + worker = RepairWorker.__new__(RepairWorker) + worker.db = _FakeDB() + worker.transfer_folder = str(tmp_path) + worker._config_manager = None + + result = worker._fix_incomplete_album( + "album", + "target-album", + None, + { + "album_id": "target-album", + "album_title": "Light Years", + "artist": "Jamiroquai", + "spotify_album_id": "sp-album", + "expected_tracks": 2, + "missing_tracks": [ + { + "name": "Light Years", + "track_number": 2, + "disc_number": 1, + "source": "spotify", + "source_track_id": "sp-light-years", + "artists": ["Jamiroquai"], + }, + ], + }, + ) + + assert result["success"] is True + assert result["fixed"] == 0 + assert result["wishlisted"] == 1 + assert worker.db.wishlist + + def test_perform_album_fill_copy_branch_generates_track_id(tmp_path, monkeypatch): src_path = tmp_path / "source.flac" src_path.write_bytes(b"fake-audio") diff --git a/tests/test_spa_deep_linking.py b/tests/test_spa_deep_linking.py index 38a90534..f57a1d90 100644 --- a/tests/test_spa_deep_linking.py +++ b/tests/test_spa_deep_linking.py @@ -95,6 +95,18 @@ class TestSpaRoutes: assert resp.status_code == 200 assert resp.data == b'INDEX_HTML' + def test_artist_detail_with_id_serves_index(self, client): + # /artist-detail/:id deep-links must serve index so the client can restore the artist. + resp = client.get('/artist-detail/42') + assert resp.status_code == 200 + assert resp.data == b'INDEX_HTML' + + def test_artist_detail_with_source_and_id_serves_index(self, client): + # /artist-detail/:source/:id is the canonical source-aware artist deep-link. + resp = client.get('/artist-detail/spotify/2YZyLoL8N0Wb9xBt1NhZWg') + assert resp.status_code == 200 + assert resp.data == b'INDEX_HTML' + # --------------------------------------------------------------------------- # Group B — Reserved prefixes are not shadowed diff --git a/tests/test_staging_album_provenance.py b/tests/test_staging_album_provenance.py new file mode 100644 index 00000000..fc62a411 --- /dev/null +++ b/tests/test_staging_album_provenance.py @@ -0,0 +1,198 @@ +"""Tests for the album-bundle provenance override in +``core/downloads/staging.py``. + +Verifies that when ``StagingDeps.get_batch_field`` returns a source +override (i.e. the batch was populated by the torrent / usenet +album-bundle flow), the staging matcher records that source on the +task instead of the generic 'staging' username. Provenance recording +downstream uses ``task['username']`` to set ``source_service`` on +the persisted download row — so this is the single point that +controls whether the history modal shows 'Torrent' / 'Usenet' vs +'Staging' / 'Soulseek'. + +Mocks the rest of StagingDeps so the test doesn't touch the +filesystem, AcoustID, or post-processing. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock + +from core.downloads.staging import StagingDeps, try_staging_match +from core.runtime_state import download_tasks, tasks_lock + + +def _make_deps(staging_file, transfer_dir, batch_field_value=None): + """Build a StagingDeps with mocked collaborators. ``batch_field_value`` + is what get_batch_field returns for ``album_bundle_source`` — None + means no override (generic staging match), 'torrent' / 'usenet' + means the album-bundle flow seeded the staging folder.""" + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = transfer_dir + return StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [staging_file], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + get_batch_field=(lambda _b, _f: batch_field_value) if batch_field_value is not None else (lambda _b, _f: None), + ) + + +def _seed_task(task_id: str, track_name: str, track_artist: str) -> None: + """Register a task in the runtime_state dict so try_staging_match + has something to mark complete.""" + with tasks_lock: + download_tasks[task_id] = { + 'status': 'searching', + 'track_info': { + 'name': track_name, + 'artists': [{'name': track_artist}], + '_is_explicit_album_download': False, + }, + } + + +def _cleanup_task(task_id: str) -> None: + with tasks_lock: + download_tasks.pop(task_id, None) + + +def test_staging_match_uses_torrent_override_when_present(tmp_path) -> None: + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value='torrent', + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_torrent_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + ok = try_staging_match(task_id, 'batch_x', track, deps) + assert ok is True + with tasks_lock: + row = download_tasks[task_id] + assert row['username'] == 'torrent', \ + f"Expected provenance override 'torrent', got {row['username']!r}" + assert row['staging_match'] is True + finally: + _cleanup_task(task_id) + + +def test_staging_match_uses_usenet_override_when_present(tmp_path) -> None: + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value='usenet', + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_usenet_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'usenet' + finally: + _cleanup_task(task_id) + + +def test_staging_match_falls_back_to_staging_without_override(tmp_path) -> None: + """When no batch override is present (manual file drop, or + batch has no album_bundle_source field), the staging matcher + uses the historical 'staging' username.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + deps = _make_deps( + staging_file={'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}, + transfer_dir=str(transfer), + batch_field_value=None, + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_no_override' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) + + +def test_staging_match_handles_missing_batch_field_callable(tmp_path) -> None: + """Backward compat: callers that build StagingDeps without + supplying ``get_batch_field`` (it defaults to None) still + work — staging matcher falls back to the 'staging' username.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = str(transfer) + deps = StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [{'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + # get_batch_field omitted — defaults to None + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_no_accessor' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) + + +def test_staging_match_swallows_accessor_exception(tmp_path) -> None: + """If the injected accessor raises (e.g. the batch was deleted + mid-process), the staging matcher should fall back to 'staging' + rather than failing the whole match.""" + src = tmp_path / 'staging' / 'gnx_track_01.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + transfer = tmp_path / 'transfer' + + def _boom(_b, _f): + raise RuntimeError("batch went away") + + me = MagicMock() + me.normalize_string.side_effect = lambda s: (s or '').lower().strip() + config = MagicMock() + config.get.return_value = str(transfer) + deps = StagingDeps( + config_manager=config, + matching_engine=me, + get_staging_file_cache=lambda _b: [{'full_path': str(src), 'title': 'Luther', 'artist': 'Kendrick Lamar'}], + docker_resolve_path=lambda p: p, + post_process_matched_download_with_verification=lambda *a, **kw: None, + get_batch_field=_boom, + ) + track = SimpleNamespace(name='Luther', artists=['Kendrick Lamar']) + task_id = 'test_task_accessor_raises' + _seed_task(task_id, 'Luther', 'Kendrick Lamar') + try: + try_staging_match(task_id, 'batch_x', track, deps) + with tasks_lock: + assert download_tasks[task_id]['username'] == 'staging' + finally: + _cleanup_task(task_id) diff --git a/tests/test_tag_writer_multi_artist.py b/tests/test_tag_writer_multi_artist.py new file mode 100644 index 00000000..c1588677 --- /dev/null +++ b/tests/test_tag_writer_multi_artist.py @@ -0,0 +1,175 @@ +"""Tests for the multi-value artist write path in tag_writer. + +Issue #587 — the AcoustID scanner's "Apply Match" retag was bypassing +the user's `metadata_enhancement.tags.write_multi_artist` setting and +writing single-string TPE1 only. The repair-path retag now passes an +``artists_list`` to ``write_tags_to_file`` and the writer respects the +config flag the same way the post-download enrichment pipeline does. +""" + +from __future__ import annotations + +import os +import tempfile +from unittest.mock import patch + +import pytest +from mutagen.flac import FLAC, StreamInfo + +from core.tag_writer import ( + _multi_artist_write_enabled, + _resolve_artists_list_for_write, + write_tags_to_file, +) + + +# ────────────────────────────────────────────────────────────────────── +# _resolve_artists_list_for_write — derives the multi-value list +# ────────────────────────────────────────────────────────────────────── + +def test_resolves_artists_list_field(): + assert _resolve_artists_list_for_write({'artists_list': ['A', 'B']}) == ['A', 'B'] + + +def test_resolves_artists_field_alias(): + assert _resolve_artists_list_for_write({'artists': ['A', 'B']}) == ['A', 'B'] + + +def test_resolves_underscore_artists_list_field(): + # _artists_list — the post-process pipeline's internal field name + assert _resolve_artists_list_for_write({'_artists_list': ['A', 'B']}) == ['A', 'B'] + + +def test_returns_none_when_no_list_supplied(): + assert _resolve_artists_list_for_write({'artist_name': 'Solo'}) is None + + +def test_returns_none_for_non_list_input(): + assert _resolve_artists_list_for_write({'artists_list': 'A, B'}) is None + assert _resolve_artists_list_for_write({'artists_list': {'A': 1}}) is None + + +def test_strips_empty_and_non_string_entries(): + out = _resolve_artists_list_for_write({'artists_list': ['A', '', None, ' ', 'B']}) + assert out == ['A', 'B'] + + +def test_returns_none_when_list_only_has_empty_entries(): + assert _resolve_artists_list_for_write({'artists_list': ['', None, ' ']}) is None + + +# ────────────────────────────────────────────────────────────────────── +# _multi_artist_write_enabled — config gate +# ────────────────────────────────────────────────────────────────────── + +def test_multi_artist_write_reads_config(): + with patch('config.settings.config_manager.get', return_value=True): + assert _multi_artist_write_enabled() is True + with patch('config.settings.config_manager.get', return_value=False): + assert _multi_artist_write_enabled() is False + + +def test_multi_artist_write_swallows_config_error(): + with patch('config.settings.config_manager.get', side_effect=RuntimeError('boom')): + assert _multi_artist_write_enabled() is False + + +# ────────────────────────────────────────────────────────────────────── +# Vorbis end-to-end — write multi-value to a real FLAC file +# ────────────────────────────────────────────────────────────────────── + +def _make_minimal_flac(path): + """Create a tiny but valid FLAC with mutagen's lowest-overhead + stream info so we can write tags and read them back.""" + # Empty audio body — just enough to satisfy mutagen's parser. + # 44-byte FLAC header + STREAMINFO block. + minimal = ( + b'fLaC' + + b'\x80\x00\x00\x22' # last STREAMINFO block, length 34 + + b'\x00\x10\x00\x10' # min/max block size + + b'\x00\x00\x00\x00\x00\x00' # min/max frame size + + b'\x0a\xc4\x42\xf0\x00\x00\x00\x00' # sample rate / channels / etc + + b'\x00' * 16 # MD5 + ) + with open(path, 'wb') as f: + f.write(minimal) + + +@pytest.fixture +def flac_path(): + fd, path = tempfile.mkstemp(suffix='.flac') + os.close(fd) + _make_minimal_flac(path) + yield path + try: + os.remove(path) + except OSError: + pass + + +def test_multi_value_artists_key_written_when_setting_on(flac_path): + with patch('config.settings.config_manager.get', return_value=True): + result = write_tags_to_file(flac_path, { + 'title': 'Track', + 'artist_name': 'Artist A, Artist B', + 'artists_list': ['Artist A', 'Artist B'], + }, embed_cover=False) + + assert result['success'] is True + assert 'artists_multi' in result['written_fields'] + + audio = FLAC(flac_path) + # Display string preserved as `artist` + assert audio.get('artist') == ['Artist A, Artist B'] + # Multi-value list written to `artists` key (Picard convention) + assert audio.get('artists') == ['Artist A', 'Artist B'] + + +def test_multi_value_artists_key_skipped_when_setting_off(flac_path): + with patch('config.settings.config_manager.get', return_value=False): + result = write_tags_to_file(flac_path, { + 'title': 'Track', + 'artist_name': 'Artist A, Artist B', + 'artists_list': ['Artist A', 'Artist B'], + }, embed_cover=False) + + assert result['success'] is True + assert 'artists_multi' not in result['written_fields'] + + audio = FLAC(flac_path) + assert audio.get('artist') == ['Artist A, Artist B'] + # No multi-value key when setting is off — backward compat + assert audio.get('artists') is None + + +def test_single_artist_does_not_write_multi_value_key(flac_path): + """When list has only one entry (or no list at all), don't + write the multi-value key even if the setting is on. The point + is to differentiate true multi-artist from single-artist tracks.""" + with patch('config.settings.config_manager.get', return_value=True): + result = write_tags_to_file(flac_path, { + 'title': 'Track', + 'artist_name': 'Solo Artist', + 'artists_list': ['Solo Artist'], + }, embed_cover=False) + + assert result['success'] is True + assert 'artists_multi' not in result['written_fields'] + audio = FLAC(flac_path) + assert audio.get('artists') is None + + +def test_no_artists_list_legacy_callers_unchanged(flac_path): + """Backward compat — callers that don't supply artists_list get + the same single-string write as before. No regression for the + write_artist_image button or any other tag_writer caller.""" + with patch('config.settings.config_manager.get', return_value=True): + result = write_tags_to_file(flac_path, { + 'title': 'Track', + 'artist_name': 'Solo Artist', + }, embed_cover=False) + + assert result['success'] is True + assert 'artists_multi' not in result['written_fields'] + audio = FLAC(flac_path) + assert audio.get('artists') is None diff --git a/tests/test_tidal_qualifier_filter.py b/tests/test_tidal_qualifier_filter.py new file mode 100644 index 00000000..e97fe51f --- /dev/null +++ b/tests/test_tidal_qualifier_filter.py @@ -0,0 +1,123 @@ +"""Tests for Tidal qualifier filtering across primary + fallback search. + +Issue #589 — when a download query carries a version qualifier ("live", +"unplugged", "acoustic", etc), the qualifier filter must apply to BOTH +the primary search AND fallback variants. Previously it only fired on +fallbacks, so a primary search for "Shy Away (MTV Unplugged Live)" that +happened to surface the studio cut first would accept the wrong file +and only get caught by AcoustID downstream. + +Also covers the album-context extension: for concert / unplugged +releases the live signal lives in the album title, not the track +title. The filter inspects both ``track.name`` AND ``track.album.name``. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from core.tidal_download_client import TidalDownloadClient + + +def _make_track(name: str, album_name: str = ''): + """Build a minimal duck-typed track object matching what the Tidal + SDK returns: a `name` attribute and an `album` attribute with its + own `name`.""" + track = MagicMock() + track.name = name + track.album = MagicMock() + track.album.name = album_name + return track + + +# ────────────────────────────────────────────────────────────────────── +# _track_name_contains_qualifiers — legacy track-only behavior preserved +# ────────────────────────────────────────────────────────────────────── + +def test_legacy_helper_passes_when_track_name_has_qualifier(): + assert TidalDownloadClient._track_name_contains_qualifiers( + 'Shy Away (MTV Unplugged Live)', ['live'] + ) is True + + +def test_legacy_helper_fails_when_track_name_lacks_qualifier(): + assert TidalDownloadClient._track_name_contains_qualifiers( + 'Shy Away', ['live'] + ) is False + + +def test_legacy_helper_passes_when_no_qualifiers_required(): + assert TidalDownloadClient._track_name_contains_qualifiers( + 'Anything', [] + ) is True + + +# ────────────────────────────────────────────────────────────────────── +# _track_matches_qualifiers — new helper inspects track + album +# ────────────────────────────────────────────────────────────────────── + +def test_qualifier_in_track_name_alone_passes(): + track = _make_track('Shy Away (Live)', 'DAMN.') + assert TidalDownloadClient._track_matches_qualifiers(track, ['live']) is True + + +def test_qualifier_in_album_name_alone_passes(): + # MTV Unplugged scenario — track titled "Shy Away" but album + # carries the live context. Pre-fix this returned False because + # only track.name was checked. + track = _make_track('Shy Away', 'MTV Unplugged Live') + assert TidalDownloadClient._track_matches_qualifiers(track, ['live']) is True + + +def test_qualifier_missing_from_both_fails(): + # User asked for live, Tidal returned the studio cut on a studio + # album. Must reject so the search keeps looking. + track = _make_track('Shy Away', 'Trench') + assert TidalDownloadClient._track_matches_qualifiers(track, ['live']) is False + + +def test_unplugged_qualifier_in_album_name(): + track = _make_track('Only If For A Night', 'MTV Unplugged') + assert TidalDownloadClient._track_matches_qualifiers(track, ['unplugged']) is True + + +def test_multiple_qualifiers_all_required(): + # Both "live" AND "acoustic" must be present somewhere + track = _make_track('Hello', 'Live Acoustic Sessions') + assert TidalDownloadClient._track_matches_qualifiers(track, ['live', 'acoustic']) is True + track2 = _make_track('Hello', 'Live Sessions') # missing acoustic + assert TidalDownloadClient._track_matches_qualifiers(track2, ['live', 'acoustic']) is False + + +def test_no_qualifiers_required_always_passes(): + track = _make_track('Anything', 'Anything') + assert TidalDownloadClient._track_matches_qualifiers(track, []) is True + + +def test_track_with_no_album_attribute(): + # Defensive — duck-typed tracks may not all have album. Use a + # plain object instead of MagicMock so missing .album is real. + class BareTrack: + name = 'Live Track' + album = None + assert TidalDownloadClient._track_matches_qualifiers(BareTrack(), ['live']) is True + assert TidalDownloadClient._track_matches_qualifiers(BareTrack(), ['unplugged']) is False + + +def test_track_with_empty_name_and_album(): + class BareTrack: + name = '' + album = None + assert TidalDownloadClient._track_matches_qualifiers(BareTrack(), ['live']) is False + + +def test_word_boundary_avoids_false_match_on_substring(): + # "session" should NOT match "obsession" + track = _make_track('Obsession', 'Pop Hits') + assert TidalDownloadClient._track_matches_qualifiers(track, ['session']) is False + + +def test_extract_qualifiers_picks_up_live_unplugged(): + quals = TidalDownloadClient._extract_qualifiers('Shy Away (MTV Unplugged Live)') + assert 'live' in quals + assert 'unplugged' in quals diff --git a/tests/test_torrent_client_adapters.py b/tests/test_torrent_client_adapters.py new file mode 100644 index 00000000..0f813b69 --- /dev/null +++ b/tests/test_torrent_client_adapters.py @@ -0,0 +1,339 @@ +"""Tests for the three torrent client adapters. + +Pins state-mapping behavior (each client has a different native state +vocabulary that must collapse onto the adapter-uniform set) and basic +HTTP / RPC plumbing so a future protocol-spec drift fails CI instead +of silently breaking downloads. +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import MagicMock, patch + +import pytest + +from core.torrent_clients import adapter_for_type, get_active_adapter +from core.torrent_clients.base import TorrentClientAdapter, TorrentStatus +from core.torrent_clients.deluge import DelugeAdapter, _map_state as deluge_map +from core.torrent_clients.qbittorrent import QBittorrentAdapter, _map_state as qbit_map +from core.torrent_clients.transmission import TransmissionAdapter, _map_state as trans_map + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def _mock_response(status_code: int, json_body=None, text=None, headers=None): + resp = MagicMock() + resp.ok = 200 <= status_code < 400 + resp.status_code = status_code + resp.headers = headers or {} + if json_body is not None: + resp.json.return_value = json_body + resp.text = text or '' + return resp + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + + +def test_adapter_for_type_returns_concrete_classes() -> None: + assert isinstance(adapter_for_type('qbittorrent'), QBittorrentAdapter) + assert isinstance(adapter_for_type('transmission'), TransmissionAdapter) + assert isinstance(adapter_for_type('deluge'), DelugeAdapter) + + +def test_adapter_for_type_returns_none_for_unknown() -> None: + assert adapter_for_type('utorrent') is None + assert adapter_for_type('') is None + + +def test_adapters_conform_to_protocol() -> None: + """``isinstance`` checks the runtime_checkable Protocol — catches + adapters that lose a required method during refactors.""" + for adapter in (QBittorrentAdapter(), TransmissionAdapter(), DelugeAdapter()): + assert isinstance(adapter, TorrentClientAdapter) + + +# --------------------------------------------------------------------------- +# State mapping +# --------------------------------------------------------------------------- + + +def test_qbittorrent_state_mapping() -> None: + assert qbit_map('downloading') == 'downloading' + assert qbit_map('forcedDL') == 'downloading' + assert qbit_map('stalledDL') == 'stalled' + assert qbit_map('uploading') == 'seeding' + assert qbit_map('pausedUP') == 'completed' + assert qbit_map('pausedDL') == 'paused' + assert qbit_map('error') == 'error' + assert qbit_map('missingFiles') == 'error' + # Unknown native value → error rather than swallowing silently. + assert qbit_map('not-a-real-state') == 'error' + + +def test_transmission_state_mapping() -> None: + assert trans_map(4, 0.5) == 'downloading' + assert trans_map(6, 1.0) == 'seeding' + # Status 0 is the ambiguous one: paused vs completed-but-not-seeding. + assert trans_map(0, 0.3) == 'paused' + assert trans_map(0, 1.0) == 'completed' + assert trans_map(2, 0.0) == 'queued' # checking files + # Unknown numeric code → error. + assert trans_map(99, 0.0) == 'error' + + +def test_deluge_state_mapping() -> None: + assert deluge_map('Downloading', 0.5) == 'downloading' + assert deluge_map('Seeding', 1.0) == 'seeding' + assert deluge_map('Paused', 0.4) == 'paused' + # Deluge reports 'Paused' for completed-not-seeding too. + assert deluge_map('Paused', 1.0) == 'completed' + assert deluge_map('Error', 0.0) == 'error' + assert deluge_map('', 0.0) == 'error' + + +# --------------------------------------------------------------------------- +# qBittorrent adapter +# --------------------------------------------------------------------------- + + +def _qbit_with_config(url='http://qbit:8080', username='admin', password='x'): + adapter = QBittorrentAdapter.__new__(QBittorrentAdapter) + import threading + adapter._session = None + adapter._session_lock = threading.Lock() + adapter._url = url.rstrip('/') + adapter._username = username + adapter._password = password + adapter._category = 'soulsync' + adapter._save_path = '' + return adapter + + +def test_qbit_is_configured_requires_only_url() -> None: + # qBittorrent allows no-auth LAN setups — URL is enough. + assert _qbit_with_config('http://x', '', '').is_configured() is True + assert _qbit_with_config('', 'u', 'p').is_configured() is False + + +def test_qbit_login_sends_referer_for_csrf() -> None: + """qBittorrent rejects login attempts without a Referer matching + its host — pin the header to catch regressions.""" + adapter = _qbit_with_config() + fake_session = MagicMock() + fake_session.post.return_value = _mock_response(200, text='Ok.') + fake_session.post.return_value.text = 'Ok.' + with patch('core.torrent_clients.qbittorrent.http_requests.Session', + return_value=fake_session): + sess = adapter._ensure_session_sync() + assert sess is not None + args, kwargs = fake_session.post.call_args + assert args[0].endswith('/api/v2/auth/login') + assert kwargs['headers']['Referer'] == 'http://qbit:8080' + assert kwargs['data'] == {'username': 'admin', 'password': 'x'} + + +def test_qbit_login_failure_returns_none() -> None: + adapter = _qbit_with_config() + fake_session = MagicMock() + bad_resp = _mock_response(200, text='Fails.') + bad_resp.text = 'Fails.' + fake_session.post.return_value = bad_resp + with patch('core.torrent_clients.qbittorrent.http_requests.Session', + return_value=fake_session): + sess = adapter._ensure_session_sync() + assert sess is None + + +def test_qbit_parse_status_normalises_native_fields() -> None: + adapter = _qbit_with_config() + status = adapter._parse_status({ + 'hash': 'abc123', 'name': 'Album', + 'state': 'downloading', 'progress': 0.5, + 'size': 1024, 'downloaded': 512, + 'dlspeed': 200, 'upspeed': 50, + 'num_seeds': 4, 'num_leechs': 1, + 'eta': 60, 'save_path': '/data/torrents', + }) + assert status == TorrentStatus( + id='abc123', name='Album', state='downloading', + progress=0.5, size=1024, downloaded=512, + download_speed=200, upload_speed=50, seeders=4, peers=1, + eta=60, save_path='/data/torrents', + ) + + +def test_qbit_parse_status_zeros_eta_when_unknown() -> None: + adapter = _qbit_with_config() + # qBittorrent uses 8640000 for "unknown" but the adapter just + # treats anything <= 0 as unknown; pin that 0 maps to None. + status = adapter._parse_status({ + 'hash': 'x', 'name': 'X', 'state': 'stalledDL', + 'progress': 0.0, 'size': 100, 'downloaded': 0, + 'dlspeed': 0, 'upspeed': 0, 'eta': 0, + }) + assert status.eta is None + + +# --------------------------------------------------------------------------- +# Transmission adapter +# --------------------------------------------------------------------------- + + +def _trans_with_config(url='http://trans:9091/transmission/rpc'): + adapter = TransmissionAdapter.__new__(TransmissionAdapter) + import threading + adapter._session_id = None + adapter._session_id_lock = threading.Lock() + adapter._url = url + adapter._username = '' + adapter._password = '' + adapter._category = 'soulsync' + adapter._save_path = '' + return adapter + + +def test_transmission_normalises_bare_host_to_rpc_path() -> None: + """Users sometimes paste ``http://host:9091``; the adapter must + append ``/transmission/rpc`` so the request hits the right + endpoint.""" + adapter = TransmissionAdapter.__new__(TransmissionAdapter) + with patch('core.torrent_clients.transmission.config_manager') as cm: + cm.get.side_effect = lambda key, default='': { + 'torrent_client.url': 'http://host:9091', + 'torrent_client.username': '', + 'torrent_client.password': '', + 'torrent_client.category': 'soulsync', + 'torrent_client.save_path': '', + }.get(key, default) + import threading + adapter._session_id = None + adapter._session_id_lock = threading.Lock() + adapter._load_config() + assert adapter._url == 'http://host:9091/transmission/rpc' + + +def test_transmission_session_id_renegotiation() -> None: + """Transmission rejects the first call with 409 and a fresh + ``X-Transmission-Session-Id`` header; the adapter must store it + and retry the same call exactly once.""" + adapter = _trans_with_config() + first = _mock_response(409, headers={'X-Transmission-Session-Id': 'sid-2'}) + second = _mock_response(200, json_body={'result': 'success', 'arguments': {'session-id': 1}}) + with patch('core.torrent_clients.transmission.http_requests.post', + side_effect=[first, second]) as mock_post: + result = adapter._rpc('session-get', {}) + assert result == {'session-id': 1} + assert mock_post.call_count == 2 + # Second call carried the new session id. + second_call_kwargs = mock_post.call_args_list[1].kwargs + assert second_call_kwargs['headers']['X-Transmission-Session-Id'] == 'sid-2' + + +def test_transmission_rpc_returns_none_on_failure_result() -> None: + adapter = _trans_with_config() + with patch('core.torrent_clients.transmission.http_requests.post', + return_value=_mock_response(200, json_body={'result': 'unknown method'})): + assert adapter._rpc('bogus', {}) is None + + +def test_transmission_add_torrent_handles_duplicate() -> None: + """torrent-add returns either ``torrent-added`` (new) or + ``torrent-duplicate`` (already-there) — both must surface the hash.""" + adapter = _trans_with_config() + with patch.object(adapter, '_rpc', return_value={'torrent-duplicate': {'hashString': 'dup'}}): + hash_id = adapter._add_torrent_sync('magnet:?xt=urn:btih:abc', 'cat', None) + assert hash_id == 'dup' + + +def test_transmission_parse_status() -> None: + adapter = _trans_with_config() + status = adapter._parse_status({ + 'hashString': 'h', 'name': 'X', 'status': 4, 'percentDone': 0.42, + 'totalSize': 100, 'downloadedEver': 42, + 'rateDownload': 10, 'rateUpload': 5, + 'peersSendingToUs': 2, 'peersGettingFromUs': 0, + 'eta': 300, 'downloadDir': '/dl', 'errorString': '', + }) + assert status.id == 'h' + assert status.state == 'downloading' + assert status.progress == 0.42 + assert status.eta == 300 + + +def test_transmission_parse_status_negative_eta_is_none() -> None: + """Transmission reports -1 / -2 for 'unknown' ETA — must normalise to None.""" + adapter = _trans_with_config() + status = adapter._parse_status({ + 'hashString': 'h', 'name': 'X', 'status': 4, 'percentDone': 0.0, + 'totalSize': 100, 'downloadedEver': 0, + 'rateDownload': 0, 'rateUpload': 0, + 'peersSendingToUs': 0, 'peersGettingFromUs': 0, + 'eta': -1, 'downloadDir': '/dl', + }) + assert status.eta is None + + +# --------------------------------------------------------------------------- +# Deluge adapter +# --------------------------------------------------------------------------- + + +def _deluge_with_config(url='http://deluge:8112', password='delugepass'): + adapter = DelugeAdapter.__new__(DelugeAdapter) + import threading + from itertools import count + adapter._session = None + adapter._session_lock = threading.Lock() + adapter._id_counter = count(1) + adapter._url = url.rstrip('/') + adapter._password = password + adapter._category = 'soulsync' + adapter._save_path = '' + return adapter + + +def test_deluge_is_configured_requires_password() -> None: + assert _deluge_with_config('http://x', '').is_configured() is False + assert _deluge_with_config('http://x', 'pw').is_configured() is True + + +def test_deluge_add_torrent_uses_magnet_method() -> None: + adapter = _deluge_with_config() + with patch.object(adapter, '_ensure_session_sync', return_value=MagicMock()), \ + patch.object(adapter, '_rpc_sync', return_value='hash123') as mock_rpc: + hash_id = adapter._add_torrent_sync('magnet:?xt=urn:btih:abc', 'cat', None) + assert hash_id == 'hash123' + # First call was core.add_torrent_magnet, not the URL variant. + first_method = mock_rpc.call_args_list[0].args[0] + assert first_method == 'core.add_torrent_magnet' + + +def test_deluge_add_torrent_uses_url_method_for_http() -> None: + adapter = _deluge_with_config() + with patch.object(adapter, '_ensure_session_sync', return_value=MagicMock()), \ + patch.object(adapter, '_rpc_sync', return_value='hash456') as mock_rpc: + hash_id = adapter._add_torrent_sync('https://example.com/x.torrent', 'cat', None) + assert hash_id == 'hash456' + first_method = mock_rpc.call_args_list[0].args[0] + assert first_method == 'core.add_torrent_url' + + +def test_deluge_parse_status_normalises_percent_progress() -> None: + """Deluge reports progress as 0-100 (not 0-1) — adapter must + normalise.""" + adapter = _deluge_with_config() + status = adapter._parse_status({ + 'hash': 'abc', 'name': 'X', 'state': 'Downloading', + 'progress': 42.0, + 'total_size': 1000, 'total_done': 420, + 'download_payload_rate': 100, 'upload_payload_rate': 0, + 'num_seeds': 1, 'num_peers': 0, 'eta': 0, + }) + assert status.progress == pytest.approx(0.42) + assert status.state == 'downloading' diff --git a/tests/test_torrent_usenet_plugins.py b/tests/test_torrent_usenet_plugins.py new file mode 100644 index 00000000..1dd9f75b --- /dev/null +++ b/tests/test_torrent_usenet_plugins.py @@ -0,0 +1,507 @@ +"""Tests for ``core/download_plugins/torrent.py`` and ``usenet.py``. + +Both plugins compose a Prowlarr client + an adapter + the archive +pipeline. The tests mock the Prowlarr client and the active adapter +factory so we can pin the projection logic, filename encoding / +decoding, finalize path, and the cancel / clear lifecycle without +touching the network or filesystem (beyond ``tmp_path``). +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Optional +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from core.download_plugins.torrent import ( + TorrentDownloadPlugin, + _adapter_state_to_display, + _decode_filename, + _FILENAME_SEP, + _guess_quality_from_title, + _parse_release_title, +) +from core.download_plugins.usenet import UsenetDownloadPlugin +from core.prowlarr_client import ProwlarrSearchResult +from core.torrent_clients.base import TorrentStatus +from core.usenet_clients.base import UsenetStatus + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + + +def test_decode_filename_splits_on_separator() -> None: + url, display = _decode_filename(f"https://x/y.torrent{_FILENAME_SEP}Album Name") + assert url == 'https://x/y.torrent' + assert display == 'Album Name' + + +def test_decode_filename_without_separator_returns_none_url() -> None: + url, display = _decode_filename('just a name') + assert url is None + assert display == 'just a name' + + +def test_decode_filename_handles_magnet_with_embedded_separators() -> None: + """Magnet URIs contain ``=`` and ``&`` but no ``||`` — so a + magnet must round-trip cleanly through the encoder.""" + magnet = 'magnet:?xt=urn:btih:abc123&dn=Album+Name' + encoded = f"{magnet}{_FILENAME_SEP}Display" + url, display = _decode_filename(encoded) + assert url == magnet + assert display == 'Display' + + +def test_guess_quality_from_title() -> None: + assert _guess_quality_from_title('Album [FLAC]') == 'flac' + assert _guess_quality_from_title('Album 24-bit Hi-Res') == 'flac' + assert _guess_quality_from_title('Album [MP3 320]') == 'mp3' + assert _guess_quality_from_title('Album [AAC 256]') == 'aac' + assert _guess_quality_from_title('Album [OGG]') == 'ogg' + # Default fallback so quality_score doesn't crash on bare titles. + assert _guess_quality_from_title('Just A Title') == 'mp3' + assert _guess_quality_from_title('') == 'mp3' + + +def test_parse_release_title_splits_artist_dash_title() -> None: + """Most release titles follow 'Artist - Title' / 'Artist - Album'.""" + assert _parse_release_title('Danny Brown - Atrocity Exhibition') == ('Danny Brown', 'Atrocity Exhibition') + assert _parse_release_title('Kendrick Lamar - DAMN.') == ('Kendrick Lamar', 'DAMN.') + + +def test_parse_release_title_strips_trailing_tags() -> None: + """Quality / year tags at the end shouldn't pollute the title.""" + artist, title = _parse_release_title('Danny Brown - Atrocity Exhibition [FLAC]') + assert artist == 'Danny Brown' + assert title == 'Atrocity Exhibition' + artist, title = _parse_release_title('Danny Brown - Atrocity Exhibition (2016)') + assert artist == 'Danny Brown' + assert title == 'Atrocity Exhibition' + + +def test_parse_release_title_handles_no_dash() -> None: + """Some indexers post bare titles. Caller should fall back to + the indexer name as the 'artist' field.""" + artist, title = _parse_release_title('JustATitle') + assert artist == '' + assert title == 'JustATitle' + + +def test_parse_release_title_handles_dashes_in_title() -> None: + """Track titles can themselves contain dashes — only split on + the FIRST one so subtitles survive.""" + artist, title = _parse_release_title('Artist - Title - Live Version') + assert artist == 'Artist' + assert title == 'Title - Live Version' + + +def test_parse_release_title_rejects_url_prefix() -> None: + """Defensive: if a URL somehow lands in the title field, refuse + to call it an artist.""" + artist, title = _parse_release_title('https://example.com/x - Album') + assert artist == '' + + +def test_adapter_state_mapping_covers_complete_states() -> None: + assert _adapter_state_to_display('downloading') == 'InProgress, Downloading' + assert _adapter_state_to_display('seeding') == 'Completed, Succeeded' + assert _adapter_state_to_display('completed') == 'Completed, Succeeded' + assert _adapter_state_to_display('error') == 'Completed, Errored' + assert _adapter_state_to_display('stalled') == 'InProgress, Stalled' + # Unknown state falls through with title-casing rather than crashing. + assert _adapter_state_to_display('weird') == 'Weird' + + +# --------------------------------------------------------------------------- +# Torrent plugin — search projection +# --------------------------------------------------------------------------- + + +def _make_torrent_result(**overrides) -> ProwlarrSearchResult: + base = dict( + guid='guid-1', title='Danny Brown - Atrocity Exhibition [FLAC]', indexer_id=3, + indexer_name='Indexer', protocol='torrent', + download_url='https://x/y.torrent', magnet_uri=None, + info_url=None, size=500_000_000, seeders=12, leechers=3, + grabs=100, publish_date='2026-01-01', categories=[3040], + raw={}, + ) + base.update(overrides) + return ProwlarrSearchResult(**base) + + +def test_torrent_project_results_drops_non_torrent_protocol() -> None: + plugin = TorrentDownloadPlugin() + results = [ + _make_torrent_result(), + _make_torrent_result(protocol='usenet', title='Usenet Album'), + ] + tracks, albums = plugin._project_results(results) + assert len(tracks) == 1 + assert tracks[0].title == 'Atrocity Exhibition' + assert tracks[0].artist == 'Danny Brown' + assert len(albums) == 1 + + +def test_torrent_project_results_drops_releases_without_download_url() -> None: + plugin = TorrentDownloadPlugin() + results = [_make_torrent_result(download_url=None, magnet_uri=None)] + tracks, albums = plugin._project_results(results) + assert tracks == [] + assert albums == [] + + +def test_torrent_project_results_prefers_magnet_when_available() -> None: + plugin = TorrentDownloadPlugin() + magnet = 'magnet:?xt=urn:btih:abc' + results = [_make_torrent_result(magnet_uri=magnet, download_url='https://x/y.torrent')] + tracks, _ = plugin._project_results(results) + url, _ = _decode_filename(tracks[0].filename) + assert url == magnet + + +def test_torrent_project_results_encodes_url_and_title_in_filename() -> None: + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result()]) + url, display = _decode_filename(tracks[0].filename) + assert url == 'https://x/y.torrent' + assert display == 'Danny Brown - Atrocity Exhibition [FLAC]' + + +def test_torrent_project_falls_back_to_indexer_name_when_title_lacks_dash() -> None: + """When the title has no 'Artist -' prefix we'd auto-parse the + filename (which starts with the indexer download URL) and end + up showing the URL in the UI's 'by' field. Pre-filling artist + with the indexer name avoids that.""" + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result(title='JustATitle')]) + assert tracks[0].artist == 'Indexer' + # And the URL is definitely not the artist. + assert 'http' not in tracks[0].artist + assert '||' not in tracks[0].artist + + +def test_torrent_project_results_neutralizes_soulseek_specific_fields() -> None: + """TrackResult.quality_score punishes results with no upload + slots; torrent results don't have that concept so the + projection has to fill in non-punishing neutral values.""" + plugin = TorrentDownloadPlugin() + tracks, _ = plugin._project_results([_make_torrent_result(seeders=0)]) + # seeders=0 means we should still hand the picker something + # usable. free_upload_slots floors at 1 to avoid the 0-slot + # penalty applied to dead Soulseek peers. + assert tracks[0].free_upload_slots >= 1 + + +# --------------------------------------------------------------------------- +# Torrent plugin — is_configured / check_connection +# --------------------------------------------------------------------------- + + +def test_torrent_is_configured_requires_both_sides() -> None: + plugin = TorrentDownloadPlugin() + with patch.object(plugin._prowlarr, 'is_configured', return_value=False), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=None): + assert plugin.is_configured() is False + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + assert plugin.is_configured() is False + fake_adapter.is_configured.return_value = True + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + assert plugin.is_configured() is True + + +# --------------------------------------------------------------------------- +# Torrent plugin — finalize / cancel / clear +# --------------------------------------------------------------------------- + + +def test_torrent_finalize_picks_first_audio_file(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + # Seed an in-flight download row + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + # Drop two audio files in the save dir + (tmp_path / 'b.flac').write_bytes(b'fLaC') + (tmp_path / 'a.mp3').write_bytes(b'ID3') + plugin._finalize_download('dl-1', str(tmp_path)) + row = plugin.active_downloads['dl-1'] + assert row['state'] == 'Completed, Succeeded' + assert row['progress'] == 100.0 + # Walker sorts → 'a.mp3' wins as first. + assert row['file_path'].endswith('a.mp3') + + +def test_torrent_finalize_marks_error_when_no_audio(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + # tmp_path has no audio files + plugin._finalize_download('dl-1', str(tmp_path)) + assert plugin.active_downloads['dl-1']['state'] == 'Completed, Errored' + assert 'No audio files' in plugin.active_downloads['dl-1']['error'] + + +def test_torrent_finalize_marks_error_when_save_path_missing() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['dl-1'] = { + 'id': 'dl-1', 'filename': 'x', 'username': 'torrent', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'torrent_hash': 'h1', 'error': None, + } + plugin._finalize_download('dl-1', None) + assert plugin.active_downloads['dl-1']['state'] == 'Completed, Errored' + assert 'no save_path' in plugin.active_downloads['dl-1']['error'].lower() + + +def test_torrent_clear_completed_drops_only_done_rows() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['a'] = {'id': 'a', 'state': 'InProgress, Downloading'} + plugin.active_downloads['b'] = {'id': 'b', 'state': 'Completed, Succeeded'} + plugin.active_downloads['c'] = {'id': 'c', 'state': 'Completed, Errored'} + plugin.active_downloads['d'] = {'id': 'd', 'state': 'Cancelled'} + _run(plugin.clear_all_completed_downloads()) + assert list(plugin.active_downloads.keys()) == ['a'] + + +def test_torrent_get_all_returns_status_objects() -> None: + plugin = TorrentDownloadPlugin() + plugin.active_downloads['a'] = { + 'id': 'a', 'filename': 'f', 'username': 'torrent', + 'state': 'InProgress, Downloading', 'progress': 50.0, + 'size': 100, 'transferred': 50, 'speed': 1000, + 'file_path': None, + } + statuses = _run(plugin.get_all_downloads()) + assert len(statuses) == 1 + assert statuses[0].id == 'a' + assert statuses[0].progress == 50.0 + + +# --------------------------------------------------------------------------- +# Usenet plugin — projection +# --------------------------------------------------------------------------- + + +def _make_usenet_result(**overrides) -> ProwlarrSearchResult: + base = dict( + guid='guid-u', title='Some Artist - Some Album', indexer_id=5, + indexer_name='UsenetIndexer', protocol='usenet', + download_url='https://x/y.nzb', magnet_uri=None, + info_url=None, size=400_000_000, seeders=None, leechers=None, + grabs=42, publish_date='2026-01-01', categories=[3010], + raw={}, + ) + base.update(overrides) + return ProwlarrSearchResult(**base) + + +def test_usenet_project_drops_torrent_protocol() -> None: + plugin = UsenetDownloadPlugin() + results = [_make_usenet_result(), _make_usenet_result(protocol='torrent', title='T')] + tracks, albums = plugin._project_results(results) + assert len(tracks) == 1 + assert tracks[0].username == 'usenet' + + +def test_usenet_project_drops_results_without_download_url() -> None: + """Usenet plugins reject magnet-only results entirely — NZBs + don't have a magnet equivalent.""" + plugin = UsenetDownloadPlugin() + results = [_make_usenet_result(download_url=None)] + tracks, _ = plugin._project_results(results) + assert tracks == [] + + +def test_usenet_project_encodes_url_in_filename() -> None: + plugin = UsenetDownloadPlugin() + tracks, _ = plugin._project_results([_make_usenet_result()]) + url, display = _decode_filename(tracks[0].filename) + assert url == 'https://x/y.nzb' + assert display == 'Some Artist - Some Album' + # Artist + title should be parsed out, not auto-extracted from filename. + assert tracks[0].artist == 'Some Artist' + assert tracks[0].title == 'Some Album' + + +def test_usenet_finalize_picks_first_audio_file(tmp_path: Path) -> None: + """Same finalize contract as torrent — sanity check the shared + helper path works for usenet too.""" + plugin = UsenetDownloadPlugin() + plugin.active_downloads['u-1'] = { + 'id': 'u-1', 'filename': 'x', 'username': 'usenet', + 'display_name': 'X', 'state': 'InProgress, Downloading', + 'progress': 50.0, 'size': 0, 'transferred': 0, 'speed': 0, + 'file_path': None, 'job_id': 'j1', 'error': None, + } + (tmp_path / 'track1.flac').write_bytes(b'fLaC') + plugin._finalize_download('u-1', str(tmp_path)) + assert plugin.active_downloads['u-1']['state'] == 'Completed, Succeeded' + assert plugin.active_downloads['u-1']['file_path'].endswith('track1.flac') + + +def test_usenet_is_configured_requires_both_sides() -> None: + plugin = UsenetDownloadPlugin() + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = True + with patch.object(plugin._prowlarr, 'is_configured', return_value=False), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=fake_adapter): + assert plugin.is_configured() is False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=None): + assert plugin.is_configured() is False + with patch.object(plugin._prowlarr, 'is_configured', return_value=True), \ + patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=fake_adapter): + assert plugin.is_configured() is True + + +# --------------------------------------------------------------------------- +# Plugin conformance — both must satisfy the DownloadSourcePlugin Protocol +# --------------------------------------------------------------------------- + + +def test_usenet_reload_settings_refreshes_cached_prowlarr_config(monkeypatch) -> None: + """Settings saves must update the plugin's held ProwlarrClient. + + The active usenet adapter is rebuilt from config on each call, but + ProwlarrClient is cached inside the plugin. This is the path that + used to require a process restart after entering Prowlarr settings. + """ + settings = { + 'prowlarr.url': '', + 'prowlarr.api_key': '', + } + monkeypatch.setattr( + 'core.prowlarr_client.config_manager.get', + lambda key, default=None: settings.get(key, default), + ) + + plugin = UsenetDownloadPlugin() + assert plugin._prowlarr.is_configured() is False + + settings.update({ + 'prowlarr.url': 'http://prowlarr:9696', + 'prowlarr.api_key': 'secret', + }) + plugin.reload_settings() + + assert plugin._prowlarr.is_configured() is True + + +def test_plugins_conform_to_protocol() -> None: + from core.download_plugins.base import DownloadSourcePlugin + assert isinstance(TorrentDownloadPlugin(), DownloadSourcePlugin) + assert isinstance(UsenetDownloadPlugin(), DownloadSourcePlugin) + + +# --------------------------------------------------------------------------- +# Registry — both should register cleanly +# --------------------------------------------------------------------------- + + +def test_torrent_album_pick_prefers_seeded_flac(tmp_path: Path) -> None: + """Album bundle picker prefers high-seeded FLAC over low-seeded MP3 + of comparable size — protects against picking a dead torrent.""" + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title + flac = _make_torrent_result(title='Kendrick Lamar - GNX [FLAC]', size=400_000_000, seeders=120) + mp3 = _make_torrent_result(title='Kendrick Lamar - GNX [MP3 320]', size=120_000_000, seeders=5, guid='guid-2') + picked = pick_best_album_release([flac, mp3], _guess_quality_from_title) + assert picked is flac + + +def test_torrent_album_pick_drops_too_small() -> None: + """Single-track torrents (~10 MB) shouldn't be picked when the user + is downloading a whole album — the size floor (40 MB) catches them.""" + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title + single = _make_torrent_result(title='Kendrick Lamar - HUMBLE', size=10_000_000, seeders=500) + album = _make_torrent_result(title='Kendrick Lamar - DAMN [MP3]', size=120_000_000, seeders=50, guid='guid-2') + picked = pick_best_album_release([single, album], _guess_quality_from_title) + assert picked is album + + +def test_torrent_album_pick_falls_back_when_all_outside_size_range() -> None: + """If every candidate is below the floor (e.g. all results are + singles), pick the most-seeded one rather than returning None — + user still wants a download even if it's a track torrent.""" + from core.download_plugins.album_bundle import pick_best_album_release + from core.download_plugins.torrent import _guess_quality_from_title + small_a = _make_torrent_result(title='X [MP3]', size=8_000_000, seeders=5) + small_b = _make_torrent_result(title='Y [MP3]', size=9_000_000, seeders=80, guid='guid-2') + picked = pick_best_album_release([small_a, small_b], _guess_quality_from_title) + assert picked is small_b + + +def test_unique_staging_path_handles_collision(tmp_path: Path) -> None: + from core.download_plugins.album_bundle import unique_staging_path + src = tmp_path / 'src' / 'track.flac' + src.parent.mkdir() + src.write_bytes(b'fLaC') + dest_dir = tmp_path / 'staging' + dest_dir.mkdir() + # First call returns the natural name. + first = unique_staging_path(dest_dir, src) + assert first == dest_dir / 'track.flac' + first.write_bytes(b'fLaC') + # Second call picks a non-colliding suffix. + second = unique_staging_path(dest_dir, src) + assert second == dest_dir / 'track_1.flac' + + +def test_torrent_album_to_staging_short_circuits_when_not_configured() -> None: + """The gate must refuse to operate when Prowlarr isn't set up — + every later call would hit the network with empty creds.""" + plugin = TorrentDownloadPlugin() + with patch.object(plugin, 'is_configured', return_value=False): + outcome = plugin.download_album_to_staging('GNX', 'Kendrick Lamar', '/tmp/staging') + assert outcome['success'] is False + assert 'not configured' in outcome['error'].lower() + + +def test_torrent_album_to_staging_ignores_candidates_without_download_url(tmp_path: Path) -> None: + plugin = TorrentDownloadPlugin() + fake_adapter = MagicMock() + fake_adapter.is_configured.return_value = True + with patch.object(plugin, 'is_configured', return_value=True), \ + patch.object(plugin._prowlarr, 'search', new=AsyncMock(return_value=[ + _make_torrent_result(download_url=None, magnet_uri=None), + ])), \ + patch('core.download_plugins.torrent.get_active_torrent_adapter', return_value=fake_adapter): + outcome = plugin.download_album_to_staging('GNX', 'Kendrick Lamar', str(tmp_path)) + + assert outcome['success'] is False + assert 'No torrent results' in outcome['error'] + fake_adapter.add_torrent.assert_not_called() + + +def test_registry_includes_torrent_and_usenet() -> None: + """The registry decides what shows up in the orchestrator's + iteration helpers. If we forget to register a new plugin the + download source dropdown will silently no-op.""" + from core.download_plugins.registry import build_default_registry + registry = build_default_registry() + names = registry.names() + assert 'torrent' in names + assert 'usenet' in names diff --git a/tests/test_unknown_artist_fixer.py b/tests/test_unknown_artist_fixer.py index 4ff15f59..5da80e8d 100644 --- a/tests/test_unknown_artist_fixer.py +++ b/tests/test_unknown_artist_fixer.py @@ -239,3 +239,61 @@ def test_unknown_artist_fixer_supports_hydrabase_title_search(monkeypatch): assert result["source"] == "hydrabase_title_search" assert hydrabase_client.search_calls == [("Hydra Match", 5)] assert spotify_client.search_calls == [] + + +# --------------------------------------------------------------------------- +# Issue #646 — deferred imports inside scan() must resolve at runtime +# --------------------------------------------------------------------------- + + +def test_deferred_path_imports_resolve(): + """Issue #646 regression guard. The Unknown Artist Fixer's scan() + defers `get_file_path_from_template_raw` + `get_audio_quality_string` + imports to keep web_server's heavy boot off the test harness — but + that means a stale import target only surfaces at *runtime*, mid- + scan, with an ImportError. The fixer crashed with exactly that: + + ImportError: cannot import name '_build_path_from_template' from + 'core.repair_jobs.library_reorganize' + + after commit ca5c9316 rewrote `library_reorganize` and moved the + helpers into the import pipeline. + + This test runs the same import statements scan() runs, so the next + refactor that moves these helpers fails CI rather than reaching the + user.""" + from core.imports.paths import get_file_path_from_template_raw # noqa: F401 + from core.imports.file_ops import get_audio_quality_string # noqa: F401 + + +def test_deferred_path_helper_shape_matches_fixer_usage(): + """Pin the shape contract the fixer relies on: pass a template + string + a context dict with the same keys scan() builds, expect a + `(folder, filename_base)` tuple back. If either of those moves, the + fixer's `folder, fname_base = ...` unpack would fail loudly here + instead of producing a malformed expected_rel path.""" + from core.imports.paths import get_file_path_from_template_raw + + template = "$albumartist/$albumartist - $album/$track - $title" + tmpl_ctx = { + "artist": "Test Artist", + "albumartist": "Test Artist", + "album": "Test Album", + "title": "Test Track", + "track_number": 1, + "disc_number": 1, + "year": "2026", + "quality": "FLAC 16bit", + "albumtype": "Album", + } + + result = get_file_path_from_template_raw(template, tmpl_ctx) + + assert isinstance(result, tuple) and len(result) == 2, \ + "Must return a 2-tuple — fixer does `folder, fname_base = result`" + folder, fname_base = result + assert isinstance(folder, str) and isinstance(fname_base, str) + # Folder path must include the album-artist segment from the template. + assert "Test Artist" in folder + # Filename base must include the title from the template. + assert "Test Track" in fname_base diff --git a/tests/test_usenet_client_adapters.py b/tests/test_usenet_client_adapters.py new file mode 100644 index 00000000..8dfb28c6 --- /dev/null +++ b/tests/test_usenet_client_adapters.py @@ -0,0 +1,297 @@ +"""Tests for the SABnzbd and NZBGet usenet adapters. + +Pins state-mapping behavior and the queue-vs-history merge logic so +get_all returns both active and completed jobs without losing +either bucket. +""" + +from __future__ import annotations + +import asyncio +import base64 +from unittest.mock import MagicMock, patch + +import pytest + +from core.usenet_clients import adapter_for_type +from core.usenet_clients.base import UsenetClientAdapter, UsenetStatus +from core.usenet_clients.nzbget import NZBGetAdapter, _map_state as nzbget_map +from core.usenet_clients.sabnzbd import SABnzbdAdapter, _map_state as sab_map + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def _mock_response(status_code: int, json_body=None): + resp = MagicMock() + resp.ok = 200 <= status_code < 400 + resp.status_code = status_code + if json_body is not None: + resp.json.return_value = json_body + return resp + + +# --------------------------------------------------------------------------- +# Factory + protocol conformance +# --------------------------------------------------------------------------- + + +def test_adapter_for_type_returns_concrete_classes() -> None: + assert isinstance(adapter_for_type('sabnzbd'), SABnzbdAdapter) + assert isinstance(adapter_for_type('nzbget'), NZBGetAdapter) + + +def test_adapter_for_type_returns_none_for_unknown() -> None: + assert adapter_for_type('unknown') is None + + +def test_adapters_conform_to_protocol() -> None: + for adapter in (SABnzbdAdapter(), NZBGetAdapter()): + assert isinstance(adapter, UsenetClientAdapter) + + +# --------------------------------------------------------------------------- +# SABnzbd +# --------------------------------------------------------------------------- + + +def _sab_with_config(url='http://sab:8080', api_key='k'): + adapter = SABnzbdAdapter.__new__(SABnzbdAdapter) + adapter._url = url.rstrip('/') + adapter._api_key = api_key + adapter._category = 'soulsync' + return adapter + + +def test_sab_is_configured_requires_url_and_key() -> None: + assert _sab_with_config('http://x', '').is_configured() is False + assert _sab_with_config('', 'k').is_configured() is False + assert _sab_with_config('http://x', 'k').is_configured() is True + + +def test_sab_state_mapping_covers_queue_states() -> None: + assert sab_map('Downloading') == 'downloading' + assert sab_map('Verifying') == 'verifying' + assert sab_map('Repairing') == 'repairing' + assert sab_map('Extracting') == 'extracting' + assert sab_map('Paused') == 'paused' + assert sab_map('Failed') == 'failed' + # Case-insensitive — SAB sometimes returns lowercase. + assert sab_map('downloading') == 'downloading' + assert sab_map('') == 'error' + + +def test_sab_parse_timeleft_handles_hhmmss() -> None: + # SABnzbd's timeleft is always HH:MM:SS (or H:MM:SS). + assert SABnzbdAdapter._parse_timeleft('01:30:00') == 5400 + assert SABnzbdAdapter._parse_timeleft('00:05:30') == 330 + assert SABnzbdAdapter._parse_timeleft('00:30:00') == 1800 + # 2-part fallback covers MM:SS for robustness. + assert SABnzbdAdapter._parse_timeleft('05:30') == 330 + assert SABnzbdAdapter._parse_timeleft('garbage') is None + assert SABnzbdAdapter._parse_timeleft('') is None + assert SABnzbdAdapter._parse_timeleft(None) is None + + +def test_sab_parse_queue_slot_converts_mb_to_bytes() -> None: + adapter = _sab_with_config() + status = adapter._parse_queue_slot({ + 'nzo_id': 'SABnzbd_nzo_1', + 'filename': 'Album.nzb', + 'status': 'Downloading', + 'percentage': '42', + 'mb': '100', + 'mbleft': '58', + 'timeleft': '0:01:00', + 'cat': 'soulsync', + }) + assert status.id == 'SABnzbd_nzo_1' + assert status.state == 'downloading' + assert status.progress == pytest.approx(0.42) + assert status.size == 100 * 1024 * 1024 + assert status.downloaded == 42 * 1024 * 1024 + assert status.eta == 60 + + +def test_sab_parse_history_slot_marks_failures() -> None: + adapter = _sab_with_config() + failed = adapter._parse_history_slot({ + 'nzo_id': 'x', 'name': 'X', 'status': 'Failed', + 'bytes': 1024, 'fail_message': 'Damaged', + }) + assert failed.state == 'failed' + assert failed.progress == 0.0 + assert failed.error == 'Damaged' + + success = adapter._parse_history_slot({ + 'nzo_id': 'y', 'name': 'Y', 'status': 'Completed', + 'bytes': 1024, 'storage': '/done', + }) + assert success.state == 'completed' + assert success.progress == 1.0 + assert success.save_path == '/done' + + +def test_sab_add_nzb_via_url_returns_first_nzo_id() -> None: + adapter = _sab_with_config() + with patch('core.usenet_clients.sabnzbd.http_requests.get', + return_value=_mock_response(200, {'status': True, 'nzo_ids': ['SABnzbd_1']})) as mock_get: + job_id = _run(adapter.add_nzb('https://example.com/x.nzb', category='cat')) + assert job_id == 'SABnzbd_1' + params = mock_get.call_args.kwargs['params'] + assert params['mode'] == 'addurl' + assert params['apikey'] == 'k' + assert params['name'] == 'https://example.com/x.nzb' + assert params['cat'] == 'cat' + + +def test_sab_add_nzb_via_bytes_uses_addfile_multipart() -> None: + adapter = _sab_with_config() + with patch('core.usenet_clients.sabnzbd.http_requests.post', + return_value=_mock_response(200, {'status': True, 'nzo_ids': ['SABnzbd_2']})) as mock_post: + job_id = _run(adapter.add_nzb(b'', category='cat')) + assert job_id == 'SABnzbd_2' + assert mock_post.call_args.kwargs['params']['mode'] == 'addfile' + files = mock_post.call_args.kwargs['files'] + assert 'name' in files + assert files['name'][1] == b'' + + +def test_sab_get_all_merges_queue_and_history() -> None: + """SAB's queue and history are separate endpoints; the adapter + must hit both so completed jobs surface in the global list.""" + adapter = _sab_with_config() + queue_resp = _mock_response(200, {'queue': {'slots': [ + {'nzo_id': 'q1', 'filename': 'A.nzb', 'status': 'Downloading', + 'percentage': '10', 'mb': '100', 'mbleft': '90', 'timeleft': '0:01:00'}, + ]}}) + history_resp = _mock_response(200, {'history': {'slots': [ + {'nzo_id': 'h1', 'name': 'B.nzb', 'status': 'Completed', 'bytes': 1024}, + ]}}) + with patch('core.usenet_clients.sabnzbd.http_requests.get', + side_effect=[queue_resp, history_resp]): + statuses = adapter._get_all_sync() + assert [s.id for s in statuses] == ['q1', 'h1'] + assert [s.state for s in statuses] == ['downloading', 'completed'] + + +# --------------------------------------------------------------------------- +# NZBGet +# --------------------------------------------------------------------------- + + +def _nzbget_with_config(url='http://nzbget:6789', username='u', password='p'): + adapter = NZBGetAdapter.__new__(NZBGetAdapter) + from itertools import count + adapter._id_counter = count(1) + adapter._url = url.rstrip('/') + adapter._username = username + adapter._password = password + adapter._category = 'soulsync' + return adapter + + +def test_nzbget_is_configured_requires_all_three() -> None: + assert _nzbget_with_config('', 'u', 'p').is_configured() is False + assert _nzbget_with_config('http://x', '', 'p').is_configured() is False + assert _nzbget_with_config('http://x', 'u', '').is_configured() is False + assert _nzbget_with_config('http://x', 'u', 'p').is_configured() is True + + +def test_nzbget_state_mapping_covers_post_process_phases() -> None: + assert nzbget_map('DOWNLOADING') == 'downloading' + assert nzbget_map('PAUSED') == 'paused' + assert nzbget_map('LOADING_PARS') == 'verifying' + assert nzbget_map('REPAIRING') == 'repairing' + assert nzbget_map('UNPACKING') == 'extracting' + assert nzbget_map('PP_FINISHED') == 'completed' + assert nzbget_map('') == 'error' + + +def test_nzbget_mb_value_prefers_64bit_split() -> None: + """NZBGet ships size as FileSizeHi << 32 | FileSizeLo for clients + that need precision past 2 GB. Prefer that over the legacy MB + field when both are present.""" + val = NZBGetAdapter._mb_value({'FileSizeLo': 1024 * 1024, 'FileSizeHi': 0, 'FileSizeMB': 999}, 'FileSize') + assert val == 1.0 + + +def test_nzbget_mb_value_falls_back_to_mb() -> None: + val = NZBGetAdapter._mb_value({'FileSizeMB': 500}, 'FileSize') + assert val == 500.0 + + +def test_nzbget_add_nzb_url_passes_through_unchanged() -> None: + adapter = _nzbget_with_config() + captured = {} + + def fake_post(url, json=None, auth=None, headers=None, timeout=None): + captured['payload'] = json + return _mock_response(200, {'result': 42}) + + with patch('core.usenet_clients.nzbget.http_requests.post', side_effect=fake_post): + job_id = _run(adapter.add_nzb('https://x/x.nzb', category='cat')) + + assert job_id == '42' + params = captured['payload']['params'] + assert params[0] == '' # NZBFilename empty when content is a URL + assert params[1] == 'https://x/x.nzb' + assert params[2] == 'cat' + + +def test_nzbget_add_nzb_bytes_base64_encodes() -> None: + adapter = _nzbget_with_config() + captured = {} + + def fake_post(url, json=None, auth=None, headers=None, timeout=None): + captured['payload'] = json + return _mock_response(200, {'result': 7}) + + with patch('core.usenet_clients.nzbget.http_requests.post', side_effect=fake_post): + _run(adapter.add_nzb(b'', category='cat')) + + params = captured['payload']['params'] + assert params[0] == 'soulsync.nzb' + assert params[1] == base64.b64encode(b'').decode('ascii') + + +def test_nzbget_remove_uses_groupfinal_when_deleting_files() -> None: + """``GroupFinalDelete`` deletes downloaded data on disk; + ``GroupDelete`` just removes the queue entry. The adapter must + pick the right one based on the ``delete_files`` flag.""" + adapter = _nzbget_with_config() + with patch.object(adapter, '_rpc_sync', return_value=True) as mock_rpc: + adapter._remove_sync('42', delete_files=True) + adapter._remove_sync('42', delete_files=False) + cmds = [c.args[1][0] for c in mock_rpc.call_args_list] + assert cmds == ['GroupFinalDelete', 'GroupDelete'] + + +def test_nzbget_parse_group_computes_progress() -> None: + adapter = _nzbget_with_config() + status = adapter._parse_group({ + 'NZBID': 99, + 'NZBName': 'Album.nzb', + 'Status': 'DOWNLOADING', + 'FileSizeLo': 200 * 1024 * 1024, 'FileSizeHi': 0, + 'RemainingSizeLo': 100 * 1024 * 1024, 'RemainingSizeHi': 0, + 'DownloadRate': 500_000, + 'DestDir': '/incomplete', + 'Category': 'soulsync', + }) + assert status.id == '99' + assert status.state == 'downloading' + assert status.size == 200 * 1024 * 1024 + assert status.downloaded == 100 * 1024 * 1024 + assert status.progress == pytest.approx(0.5) + assert status.download_speed == 500_000 + + +def test_nzbget_remove_rejects_non_numeric_id() -> None: + """NZBGet IDs are ints; passing a string id like 'abc' must + fail fast instead of corrupting the editqueue call.""" + adapter = _nzbget_with_config() + with patch.object(adapter, '_rpc_sync') as mock_rpc: + assert adapter._remove_sync('not-a-number', delete_files=False) is False + mock_rpc.assert_not_called() diff --git a/tests/test_watchlist_bulk_add.py b/tests/test_watchlist_bulk_add.py index b193dd7c..19f86b42 100644 --- a/tests/test_watchlist_bulk_add.py +++ b/tests/test_watchlist_bulk_add.py @@ -69,6 +69,23 @@ def test_falls_back_to_discogs_as_last_resort() -> None: assert pick(artist) == ('dg-999', 'discogs') +def test_falls_back_to_musicbrainz_after_other_sources() -> None: + pick = _make_picker('spotify') + artist = { + 'musicbrainz_id': 'mb-999', + } + assert pick(artist) == ('mb-999', 'musicbrainz') + + +def test_active_source_musicbrainz_picks_musicbrainz_first() -> None: + pick = _make_picker('musicbrainz') + artist = { + 'spotify_artist_id': 'sp-123', + 'musicbrainz_id': 'mb-999', + } + assert pick(artist) == ('mb-999', 'musicbrainz') + + def test_returns_none_when_artist_has_zero_source_ids() -> None: """Drop only when the artist has no source IDs at all — that's the only legitimate skip reason now.""" diff --git a/tests/test_worker_utils_album_track_count.py b/tests/test_worker_utils_album_track_count.py index 4b988d70..cf96b535 100644 --- a/tests/test_worker_utils_album_track_count.py +++ b/tests/test_worker_utils_album_track_count.py @@ -1,6 +1,8 @@ """Tests for `worker_utils.set_album_api_track_count` — the shared helper enrichment workers call to cache authoritative track counts.""" +import sqlite3 + from core.worker_utils import set_album_api_track_count @@ -114,3 +116,21 @@ def test_swallows_cursor_execute_errors(): cursor = _BrokenCursor() # Should not raise. set_album_api_track_count(cursor, "album-z", 10) + + +def test_repairs_missing_api_track_count_column(): + conn = sqlite3.connect(":memory:") + cursor = conn.cursor() + cursor.execute("CREATE TABLE albums (id TEXT PRIMARY KEY, title TEXT)") + cursor.execute("INSERT INTO albums (id, title) VALUES ('album-z', 'Album')") + + set_album_api_track_count(cursor, "album-z", 10) + + cursor.execute("PRAGMA table_info(albums)") + cols = {row[1] for row in cursor.fetchall()} + cursor.execute("SELECT api_track_count FROM albums WHERE id = 'album-z'") + row = cursor.fetchone() + conn.close() + + assert 'api_track_count' in cols + assert row[0] == 10 diff --git a/tests/tools/__init__.py b/tests/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/tools/test_amazon_client.py b/tests/tools/test_amazon_client.py new file mode 100644 index 00000000..33fbac08 --- /dev/null +++ b/tests/tools/test_amazon_client.py @@ -0,0 +1,1033 @@ +"""Unit tests for core/amazon_client.py. + +All network I/O is mocked via a fake session — no real T2Tunes instance needed. + +Run from project root: + python -m pytest tests/tools/test_amazon_client.py -v +""" + +from __future__ import annotations + +import json +import sys +import threading +from pathlib import Path +from typing import Any, Dict, Optional +from unittest.mock import MagicMock, patch + +import pytest + +# Make sure project root is importable when running tests directly. +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from core.amazon_client import ( + Album, + AmazonClient, + AmazonClientError, + Artist, + T2TunesSearchItem, + T2TunesStreamInfo, + Track, + _rate_limit, +) + + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + +TRACK_DOC = { + "asin": "B09XYZ1234", + "title": "Not Like Us", + "artistName": "Kendrick Lamar", + "__type": "track", + "albumName": "GNX", + "albumAsin": "B0ABCDE123", + "duration": 217, + "isrc": "USRC12345678", +} + +ALBUM_DOC = { + "asin": "B0ABCDE123", + "albumAsin": "B0ABCDE123", + "title": "GNX", + "albumName": "GNX", + "artistName": "Kendrick Lamar", + "__type": "album", + "duration": 0, +} + +SEARCH_RESPONSE_TRACKS = { + "results": [ + { + "hits": [ + {"document": TRACK_DOC}, + { + "document": { + "asin": "B09XYZ5678", + "title": "euphoria", + "artistName": "Kendrick Lamar", + "__type": "track", + "albumName": "euphoria", + "albumAsin": "B0ABCDE456", + "duration": 480, + "isrc": "USRC87654321", + } + }, + ] + } + ] +} + +SEARCH_RESPONSE_ALBUMS = { + "results": [{"hits": [{"document": ALBUM_DOC}]}] +} + +SEARCH_RESPONSE_MIXED = { + "results": [ + { + "hits": [ + {"document": TRACK_DOC}, + {"document": ALBUM_DOC}, + ] + } + ] +} + +ALBUM_METADATA_RESPONSE = { + "albumList": [ + { + "asin": "B0ABCDE123", + "title": "GNX", + "image": "https://example.com/cover.jpg", + "trackCount": 12, + "label": "pgLang/Interscope", + "artistName": "Kendrick Lamar", + } + ] +} + +MEDIA_RESPONSE_FLAC = { + "asin": "B09XYZ1234", + "streamable": True, + "decryptionKey": None, + "streamInfo": { + "codec": "FLAC", + "format": "FLAC", + "sampleRate": 44100, + "streamUrl": "https://cdn.example.com/track.flac", + }, + "tags": { + "title": "Not Like Us", + "artist": "Kendrick Lamar", + "album": "GNX", + "isrc": "USRC12345678", + "trackNumber": "3", + "discNumber": "1", + "date": "2024-11-22", + }, +} + +MEDIA_RESPONSE_HIRES = { + "asin": "B09XYZ1234", + "streamable": True, + "decryptionKey": "somekey", + "streamInfo": { + "codec": "FLAC", + "format": "FLAC", + "sampleRate": 96000, + "streamUrl": "https://cdn.example.com/track-hires.flac", + }, + "tags": { + "title": "Not Like Us", + "artist": "Kendrick Lamar", + "album": "GNX", + "isrc": "USRC12345678", + }, +} + +STATUS_UP = {"amazonMusic": "up", "version": "1.0"} +STATUS_DOWN = {"amazonMusic": "down", "version": "1.0"} + + +def _mock_response(data: Any, status_code: int = 200) -> MagicMock: + resp = MagicMock() + resp.status_code = status_code + resp.ok = 200 <= status_code < 400 + resp.json.return_value = data + resp.text = json.dumps(data) + if status_code >= 400: + from requests import HTTPError + exc = HTTPError(response=resp) + resp.raise_for_status.side_effect = exc + else: + resp.raise_for_status.return_value = None + return resp + + +def _make_client(response_map: Optional[Dict[str, Any]] = None) -> AmazonClient: + """Build an AmazonClient with a fake session. + + response_map: path substring → response data (first match wins). + """ + session = MagicMock() + + def _get(url, params=None, timeout=None, **_): + if response_map: + for key, data in response_map.items(): + if key in url: + if isinstance(data, Exception): + raise data + return _mock_response(data) + return _mock_response({"error": "no mock for " + url}, 404) + + session.get.side_effect = _get + with patch("core.amazon_client._rate_limit"): + with patch("core.amazon_client.config_manager") as cfg: + cfg.get.return_value = "" + client = AmazonClient( + base_url="https://test.t2tunes.local", + country="US", + session=session, + ) + client.session = session + return client + + +# --------------------------------------------------------------------------- +# Dataclass construction +# --------------------------------------------------------------------------- + +class TestTrackDataclass: + def test_from_search_hit_basic(self): + t = Track.from_search_hit(TRACK_DOC) + assert t.id == "B09XYZ1234" + assert t.name == "Not Like Us" + assert t.artists == ["Kendrick Lamar"] + assert t.album == "GNX" + assert t.duration_ms == 217_000 + assert t.isrc == "USRC12345678" + assert t.popularity == 0 + + def test_from_search_hit_missing_fields(self): + t = Track.from_search_hit({}) + assert t.id == "" + assert t.name == "" + assert t.artists == ["Unknown Artist"] + assert t.duration_ms == 0 + assert t.isrc is None + + def test_from_stream_info(self): + stream = T2TunesStreamInfo( + asin="B09XYZ1234", + streamable=True, + codec="FLAC", + format="FLAC", + sample_rate=44100, + stream_url="https://cdn.example.com/track.flac", + decryption_key=None, + title="Not Like Us", + artist="Kendrick Lamar", + album="GNX", + isrc="USRC12345678", + ) + t = Track.from_stream_info(stream) + assert t.id == "B09XYZ1234" + assert t.name == "Not Like Us" + assert t.artists == ["Kendrick Lamar"] + assert t.isrc == "USRC12345678" + + def test_from_stream_info_empty_artist(self): + stream = T2TunesStreamInfo( + asin="B1", + streamable=True, + codec="FLAC", + format="FLAC", + sample_rate=44100, + stream_url="https://cdn.example.com/t.flac", + decryption_key=None, + ) + t = Track.from_stream_info(stream) + assert t.artists == ["Unknown Artist"] + + +class TestArtistDataclass: + def test_from_name(self): + a = Artist.from_name("Kendrick Lamar") + assert a.id == "kendrick_lamar" + assert a.name == "Kendrick Lamar" + assert a.genres == [] + assert a.followers == 0 + + def test_from_name_special_chars(self): + a = Artist.from_name("AC/DC") + assert "ac" in a.id + + +class TestAlbumDataclass: + def test_from_search_hit(self): + al = Album.from_search_hit(ALBUM_DOC) + assert al.id == "B0ABCDE123" + assert al.name == "GNX" + assert al.artists == ["Kendrick Lamar"] + assert al.album_type == "album" + + def test_from_search_hit_fallback_asin(self): + al = Album.from_search_hit({"asin": "B0001", "albumName": "Test", "artistName": "X"}) + assert al.id == "B0001" + + def test_from_metadata(self): + meta = ALBUM_METADATA_RESPONSE["albumList"][0] + al = Album.from_metadata(meta, asin="B0ABCDE123") + assert al.id == "B0ABCDE123" + assert al.name == "GNX" + assert al.total_tracks == 12 + assert al.image_url == "https://example.com/cover.jpg" + + +# --------------------------------------------------------------------------- +# T2TunesSearchItem helpers +# --------------------------------------------------------------------------- + +class TestT2TunesSearchItem: + def test_is_track(self): + item = T2TunesSearchItem( + asin="A1", title="T", artist_name="X", item_type="MusicTrack" + ) + assert item.is_track is True + assert item.is_album is False + + def test_is_album(self): + item = T2TunesSearchItem( + asin="A1", title="T", artist_name="X", item_type="MusicAlbum" + ) + assert item.is_album is True + assert item.is_track is False + + def test_ambiguous_type(self): + item = T2TunesSearchItem( + asin="A1", title="T", artist_name="X", item_type="Unknown" + ) + assert item.is_track is False + assert item.is_album is False + + +# --------------------------------------------------------------------------- +# _iter_search_items static method +# --------------------------------------------------------------------------- + +class TestIterSearchItems: + def test_parses_tracks(self): + items = list(AmazonClient._iter_search_items(SEARCH_RESPONSE_TRACKS)) + assert len(items) == 2 + assert items[0].asin == "B09XYZ1234" + assert items[0].title == "Not Like Us" + assert items[0].is_track + + def test_parses_albums(self): + items = list(AmazonClient._iter_search_items(SEARCH_RESPONSE_ALBUMS)) + assert len(items) == 1 + assert items[0].is_album + + def test_skips_missing_asin(self): + resp = {"results": [{"hits": [{"document": {"title": "No ASIN"}}]}]} + items = list(AmazonClient._iter_search_items(resp)) + assert items == [] + + def test_empty_results(self): + items = list(AmazonClient._iter_search_items({"results": []})) + assert items == [] + + def test_wrong_type_raises(self): + with pytest.raises(AmazonClientError): + list(AmazonClient._iter_search_items(["not", "a", "dict"])) + + def test_skips_malformed_hits(self): + resp = { + "results": [ + { + "hits": [ + "not_a_dict", + {"document": None}, + {"document": {"asin": "B1", "__type": "track", "title": "T", "artistName": "A"}}, + ] + } + ] + } + items = list(AmazonClient._iter_search_items(resp)) + assert len(items) == 1 + assert items[0].asin == "B1" + + +# --------------------------------------------------------------------------- +# _parse_stream_info static method +# --------------------------------------------------------------------------- + +class TestParseStreamInfo: + def test_flac_stream(self): + s = AmazonClient._parse_stream_info(MEDIA_RESPONSE_FLAC) + assert s.asin == "B09XYZ1234" + assert s.streamable is True + assert s.codec == "FLAC" + assert s.sample_rate == 44100 + assert s.stream_url == "https://cdn.example.com/track.flac" + assert s.has_decryption_key is False + assert s.title == "Not Like Us" + assert s.isrc == "USRC12345678" + + def test_hires_with_key(self): + s = AmazonClient._parse_stream_info(MEDIA_RESPONSE_HIRES) + assert s.sample_rate == 96000 + assert s.has_decryption_key is True + + def test_typo_stremeable(self): + data = { + "asin": "B1", + "stremeable": True, # typo variant + "streamInfo": {"codec": "OPUS", "format": "OPUS", "streamUrl": "https://x.com/t.opus"}, + "tags": {}, + } + s = AmazonClient._parse_stream_info(data) + assert s.streamable is True + assert s.codec == "OPUS" + + def test_missing_stream_info(self): + s = AmazonClient._parse_stream_info({"asin": "B1"}) + assert s.stream_url == "" + assert s.codec == "" + assert s.sample_rate is None + assert s.has_decryption_key is False + + +# --------------------------------------------------------------------------- +# AmazonClient — HTTP layer +# --------------------------------------------------------------------------- + +class TestStatus: + def test_success(self): + client = _make_client({"/api/status": STATUS_UP}) + with patch("core.amazon_client._rate_limit"): + result = client.status() + assert result["amazonMusic"] == "up" + + def test_http_error_raises(self): + client = _make_client() + client.session.get.side_effect = None + client.session.get.return_value = _mock_response({}, 503) + with pytest.raises(AmazonClientError, match="HTTP 503"): + client.status() + + def test_non_json_raises(self): + resp = MagicMock() + resp.raise_for_status.return_value = None + resp.json.side_effect = ValueError("not json") + resp.text = "error" + client = _make_client() + client.session.get.side_effect = None + client.session.get.return_value = resp + with pytest.raises(AmazonClientError, match="not JSON"): + client.status() + + +class TestIsAuthenticated: + def test_true_when_up(self): + client = _make_client({"/api/status": STATUS_UP}) + assert client.is_authenticated() is True + + def test_false_when_down(self): + client = _make_client({"/api/status": STATUS_DOWN}) + assert client.is_authenticated() is False + + def test_false_on_error(self): + from requests import RequestException + client = _make_client() + client.session.get.side_effect = RequestException("network error") + assert client.is_authenticated() is False + + +class TestReloadConfig: + def test_reloads_fields(self): + with patch("core.amazon_client.config_manager") as cfg: + cfg.get.side_effect = lambda key, default="": { + "amazon.base_url": "https://new.instance.local", + "amazon.country": "GB", + "amazon.preferred_codec": "opus", + }.get(key, default) + client = AmazonClient(session=MagicMock()) + client.reload_config() + assert "new.instance.local" in client.base_url + assert client.country == "GB" + assert client.preferred_codec == "opus" + + +# --------------------------------------------------------------------------- +# AmazonClient — search_raw +# --------------------------------------------------------------------------- + +class TestSearchRaw: + def test_returns_items(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + items = client.search_raw("Kendrick Lamar") + assert len(items) == 2 + + def test_passes_country(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + client.search_raw("test", types="track") + call_kwargs = client.session.get.call_args + assert "country" in str(call_kwargs) + + def test_network_error_raises(self): + from requests import RequestException + client = _make_client() + client.session.get.side_effect = RequestException("timeout") + with pytest.raises(AmazonClientError): + client.search_raw("test") + + +# --------------------------------------------------------------------------- +# AmazonClient — search_tracks / search_artists / search_albums +# --------------------------------------------------------------------------- + +class TestSearchTracks: + def test_returns_track_list(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + tracks = client.search_tracks("Kendrick Lamar") + assert len(tracks) == 2 + assert all(isinstance(t, Track) for t in tracks) + + def test_respects_limit(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + tracks = client.search_tracks("Kendrick Lamar", limit=1) + assert len(tracks) == 1 + + def test_ignores_album_hits(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_ALBUMS}) + with patch("core.amazon_client._rate_limit"): + tracks = client.search_tracks("GNX") + assert tracks == [] + + def test_track_fields(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + tracks = client.search_tracks("Kendrick") + t = tracks[0] + assert t.name == "Not Like Us" + assert t.artists == ["Kendrick Lamar"] + assert t.album == "GNX" + assert t.duration_ms == 217_000 + assert t.isrc == "USRC12345678" + + +class TestSearchArtists: + def test_returns_unique_artists(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Kendrick") + # Both tracks are by Kendrick Lamar — should deduplicate + assert len(artists) == 1 + assert artists[0].name == "Kendrick Lamar" + + def test_returns_artist_dataclass(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Kendrick") + assert isinstance(artists[0], Artist) + + def test_artist_image_from_album(self): + resp = { + "results": [{"hits": [ + {"document": {"asin": "A1", "title": "T1", "artistName": "Kendrick Lamar", + "__type": "track", "albumAsin": "B0ABCDE123"}}, + ]}] + } + client = _make_client({ + "amazon-music/search": resp, + "amazon-music/metadata": ALBUM_METADATA_RESPONSE, + }) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Kendrick") + assert artists[0].image_url == "https://example.com/cover.jpg" + + def test_deduplicates_feat_credits(self): + resp = { + "results": [ + { + "hits": [ + {"document": {"asin": "A1", "title": "T1", "artistName": "Kendrick Lamar", "__type": "track"}}, + {"document": {"asin": "A2", "title": "T2", "artistName": "Kendrick Lamar feat. SZA", "__type": "track"}}, + {"document": {"asin": "A3", "title": "T3", "artistName": "Kendrick Lamar ft. Drake", "__type": "track"}}, + {"document": {"asin": "A4", "title": "T4", "artistName": "SZA featuring Kendrick Lamar", "__type": "track"}}, + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Kendrick") + names = [a.name for a in artists] + assert "Kendrick Lamar" in names + assert "SZA" in names + assert len(artists) == 2 + + def test_respects_limit(self): + resp = { + "results": [ + { + "hits": [ + { + "document": { + "asin": f"B{i}", + "title": f"Song {i}", + "artistName": f"Artist {i}", + "__type": "track", + } + } + for i in range(10) + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Various", limit=3) + assert len(artists) == 3 + + +class TestSearchAlbums: + def test_returns_albums(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_ALBUMS}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + assert len(albums) == 1 + assert isinstance(albums[0], Album) + assert albums[0].id == "B0ABCDE123" + + def test_deduplicates_by_asin(self): + resp = { + "results": [ + { + "hits": [ + {"document": {**ALBUM_DOC}}, + {"document": {**ALBUM_DOC}}, # duplicate + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + assert len(albums) == 1 + + def test_derives_albums_from_track_hits(self): + """search_albums now intentionally queries `types=track` and derives + Album objects from the album metadata carried on each track hit — + Amazon's album-type query is broken upstream, so the t2tunes fix + switched everything to track-type and reconstructs albums from the + results. Distinct album ASINs across the track hits yield distinct + albums; duplicates collapse via the explicit/clean dedup key.""" + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("Kendrick") + # Two track hits → two distinct album ASINs → two derived albums. + assert {a.id for a in albums} == {"B0ABCDE123", "B0ABCDE456"} + assert {a.name for a in albums} == {"GNX", "euphoria"} + assert all(a.artists == ["Kendrick Lamar"] for a in albums) + + def test_strips_explicit_from_album_name(self): + resp = { + "results": [{"hits": [ + {"document": {**ALBUM_DOC, "albumName": "GNX (Explicit)", "title": "GNX (Explicit)"}}, + ]}] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + assert albums[0].name == "GNX" + + def test_keeps_clean_suffix(self): + resp = { + "results": [{"hits": [ + {"document": {**ALBUM_DOC, "albumName": "GNX [Clean]", "title": "GNX [Clean]"}}, + ]}] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + assert albums[0].name == "GNX [Clean]" + + def test_deduplicates_explicit_clean_as_separate(self): + resp = { + "results": [{"hits": [ + {"document": {**ALBUM_DOC, "asin": "B1", "albumAsin": "B1", "albumName": "GNX (Explicit)", "title": "GNX (Explicit)"}}, + {"document": {**ALBUM_DOC, "asin": "B2", "albumAsin": "B2", "albumName": "GNX [Clean]", "title": "GNX [Clean]"}}, + ]}] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + names = [a.name for a in albums] + assert "GNX" in names # explicit stripped + assert "GNX [Clean]" in names + assert len(albums) == 2 + + +# --------------------------------------------------------------------------- +# AmazonClient — album_metadata / media_from_asin +# --------------------------------------------------------------------------- + +class TestAlbumMetadata: + def test_returns_dict(self): + client = _make_client({"amazon-music/metadata": ALBUM_METADATA_RESPONSE}) + with patch("core.amazon_client._rate_limit"): + meta = client.album_metadata("B0ABCDE123") + assert "albumList" in meta + assert meta["albumList"][0]["title"] == "GNX" + + +class TestMediaFromAsin: + def test_list_response(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + streams = client.media_from_asin("B09XYZ1234") + assert len(streams) == 1 + assert isinstance(streams[0], T2TunesStreamInfo) + assert streams[0].codec == "FLAC" + + def test_single_dict_response(self): + client = _make_client({"amazon-music/media-from-asin": MEDIA_RESPONSE_FLAC}) + with patch("core.amazon_client._rate_limit"): + streams = client.media_from_asin("B09XYZ1234") + assert len(streams) == 1 + + def test_invalid_response_raises(self): + client = _make_client({"amazon-music/media-from-asin": "not a list or dict"}) + with pytest.raises(AmazonClientError, match="Unexpected media"): + client.media_from_asin("B09XYZ1234") + + def test_uses_preferred_codec(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + client.preferred_codec = "opus" + with patch("core.amazon_client._rate_limit"): + client.media_from_asin("B09XYZ1234") + call_kwargs = str(client.session.get.call_args) + assert "opus" in call_kwargs + + def test_codec_override(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + client.media_from_asin("B09XYZ1234", codec="eac3") + call_kwargs = str(client.session.get.call_args) + assert "eac3" in call_kwargs + + +# --------------------------------------------------------------------------- +# AmazonClient — higher-level get_* methods +# --------------------------------------------------------------------------- + +class TestGetTrackDetails: + def _client(self): + return _make_client({ + "amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC], + "amazon-music/metadata": ALBUM_METADATA_RESPONSE, + }) + + def test_returns_spotify_compat_dict(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + details = client.get_track_details("B09XYZ1234") + assert details is not None + assert details["id"] == "B09XYZ1234" + assert details["name"] == "Not Like Us" + assert "artists" in details + assert "album" in details + assert details["is_album_track"] is True + + def test_album_image_populated(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + details = client.get_track_details("B09XYZ1234") + assert details["album"]["images"][0]["url"] == "https://example.com/cover.jpg" + + def test_raw_data_present(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + details = client.get_track_details("B09XYZ1234") + assert "raw_data" in details + assert details["raw_data"]["codec"] == "FLAC" + assert details["raw_data"]["sample_rate"] == 44100 + + def test_returns_none_on_empty_streams(self): + client = _make_client({"amazon-music/media-from-asin": []}) + with patch("core.amazon_client._rate_limit"): + assert client.get_track_details("B09XYZ1234") is None + + def test_returns_none_on_api_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_track_details("B09XYZ1234") is None + + def test_graceful_when_metadata_fails(self): + session = MagicMock() + call_count = {"n": 0} + + def _get(url, params=None, timeout=None, **_): + call_count["n"] += 1 + if "media-from-asin" in url: + return _mock_response([MEDIA_RESPONSE_FLAC]) + return _mock_response({}, 500) + + session.get.side_effect = _get + client = AmazonClient(base_url="https://test.local", session=session) + with patch("core.amazon_client._rate_limit"): + details = client.get_track_details("B09XYZ1234") + assert details is not None + assert details["album"]["images"] == [] + + +class TestGetAlbum: + def _client(self): + return _make_client({"amazon-music/metadata": ALBUM_METADATA_RESPONSE, + "amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + + def test_returns_album_dict(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + album = client.get_album("B0ABCDE123") + assert album is not None + assert album["id"] == "B0ABCDE123" + assert album["name"] == "GNX" + assert album["total_tracks"] == 12 + assert album["label"] == "pgLang/Interscope" + + def test_includes_tracks_by_default(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + album = client.get_album("B0ABCDE123") + assert "tracks" in album + assert isinstance(album["tracks"], dict) + assert "items" in album["tracks"] + + def test_excludes_tracks_when_flag_false(self): + client = _make_client({"amazon-music/metadata": ALBUM_METADATA_RESPONSE}) + with patch("core.amazon_client._rate_limit"): + album = client.get_album("B0ABCDE123", include_tracks=False) + assert "tracks" not in album + + def test_release_date_backfilled_from_stream_tags(self): + # ALBUM_METADATA_RESPONSE has no release_date — should fall back to track date tag + client = self._client() + with patch("core.amazon_client._rate_limit"): + album = client.get_album("B0ABCDE123") + assert album["release_date"] == "2024-11-22" + + def test_returns_none_on_empty_albumlist(self): + client = _make_client({"amazon-music/metadata": {"albumList": []}}) + with patch("core.amazon_client._rate_limit"): + assert client.get_album("B0ABCDE123") is None + + def test_returns_none_on_api_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_album("B0ABCDE123") is None + + +class TestGetAlbumTracks: + def test_returns_spotify_pagination(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + result = client.get_album_tracks("B09XYZ1234") + assert result is not None + assert "items" in result + assert result["total"] == 1 + assert result["next"] is None + assert result["limit"] == 50 + + def test_item_fields(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + result = client.get_album_tracks("B09XYZ1234") + item = result["items"][0] + assert item["id"] == "B09XYZ1234" + assert item["name"] == "Not Like Us" + assert item["isrc"] == "USRC12345678" + assert item["track_number"] == 3 + assert item["disc_number"] == 1 + + def test_duration_enriched_from_search(self): + search_resp = { + "results": [{"hits": [ + {"document": { + "asin": "B09XYZ1234", "__type": "track", + "title": "Not Like Us", "artistName": "Kendrick Lamar", + "albumAsin": "B09XYZ1234", "duration": 217, + }}, + ]}] + } + client = _make_client({ + "amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC], + "amazon-music/search": search_resp, + }) + with patch("core.amazon_client._rate_limit"): + result = client.get_album_tracks("B09XYZ1234") + assert result["items"][0]["duration_ms"] == 217_000 + + def test_duration_zero_when_search_fails(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + result = client.get_album_tracks("B09XYZ1234") + assert result["items"][0]["duration_ms"] == 0 + + def test_returns_none_on_api_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_album_tracks("B09XYZ1234") is None + + +class TestGetArtist: + def test_returns_artist_dict(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + artist = client.get_artist("Kendrick Lamar") + assert artist is not None + assert artist["name"] == "Kendrick Lamar" + assert "genres" in artist + assert "followers" in artist + + def test_exact_match_preferred(self): + resp = { + "results": [ + { + "hits": [ + {"document": {"asin": "A1", "title": "T1", "artistName": "Kendrick Lamar", "__type": "track"}}, + {"document": {"asin": "A2", "title": "T2", "artistName": "Kendrick Lamar Jr.", "__type": "track"}}, + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + artist = client.get_artist("Kendrick Lamar") + assert artist["name"] == "Kendrick Lamar" + + def test_returns_none_when_no_match(self): + client = _make_client({"amazon-music/search": {"results": []}}) + with patch("core.amazon_client._rate_limit"): + assert client.get_artist("Nobody") is None + + def test_returns_none_on_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_artist("Kendrick") is None + + +class TestGetArtistAlbums: + def test_returns_filtered_albums(self): + resp = { + "results": [ + { + "hits": [ + { + "document": { + "asin": "B0ABCDE123", + "albumAsin": "B0ABCDE123", + "title": "GNX", + "albumName": "GNX", + "artistName": "Kendrick Lamar", + "__type": "album", + } + }, + { + "document": { + "asin": "B0ZZZ", + "albumAsin": "B0ZZZ", + "title": "Other Album", + "albumName": "Other Album", + "artistName": "Another Artist", + "__type": "album", + } + }, + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.get_artist_albums("Kendrick Lamar") + assert len(albums) == 1 + assert albums[0].name == "GNX" + + def test_respects_limit(self): + hits = [ + { + "document": { + "asin": f"B{i}", + "albumAsin": f"B{i}", + "albumName": f"Album {i}", + "artistName": "Kendrick Lamar", + "__type": "album", + } + } + for i in range(20) + ] + client = _make_client({"amazon-music/search": {"results": [{"hits": hits}]}}) + with patch("core.amazon_client._rate_limit"): + albums = client.get_artist_albums("Kendrick Lamar", limit=5) + assert len(albums) == 5 + + def test_returns_empty_on_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_artist_albums("Kendrick") == [] + + +class TestGetTrackFeatures: + def test_always_none(self): + client = AmazonClient(session=MagicMock()) + assert client.get_track_features("B09XYZ1234") is None + + +# --------------------------------------------------------------------------- +# Rate-limit enforcement +# --------------------------------------------------------------------------- + +class TestRateLimit: + def test_enforces_min_interval(self): + import core.amazon_client as mod + original = mod._last_api_call + sleeps = [] + + def fake_sleep(t): + sleeps.append(t) + + with patch("core.amazon_client.time") as mock_time: + mock_time.monotonic.return_value = mod._last_api_call + 0.1 + mock_time.sleep = fake_sleep + with patch("core.amazon_client.api_call_tracker"): + _rate_limit() + # Should have slept since interval not elapsed + assert len(sleeps) > 0 + mod._last_api_call = original + + def test_no_sleep_when_interval_elapsed(self): + import core.amazon_client as mod + original = mod._last_api_call + sleeps = [] + + with patch("core.amazon_client.time") as mock_time: + mock_time.monotonic.return_value = mod._last_api_call + 10.0 + mock_time.sleep = lambda t: sleeps.append(t) + with patch("core.amazon_client.api_call_tracker"): + _rate_limit() + assert sleeps == [] + mod._last_api_call = original diff --git a/tests/tools/test_amazon_download_client.py b/tests/tools/test_amazon_download_client.py new file mode 100644 index 00000000..f831f268 --- /dev/null +++ b/tests/tools/test_amazon_download_client.py @@ -0,0 +1,847 @@ +"""Unit tests for core/amazon_download_client.py. + +All network I/O and subprocess calls are mocked. +No real T2Tunes instance or ffmpeg binary required. + +Run from project root: + python -m pytest tests/tools/test_amazon_download_client.py -v +""" + +from __future__ import annotations + +import asyncio +import io +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from core.amazon_client import AmazonClientError, T2TunesStreamInfo +from core.amazon_download_client import ( + AmazonDownloadClient, + MIN_AUDIO_BYTES, + _codec_key, + _file_extension, + _quality_label, +) +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +def run(coro): + return asyncio.run(coro) + + +def _stream_info( + *, + asin: str = "B09XYZ1234", + streamable: bool = True, + codec: str = "FLAC", + sample_rate: int = 44100, + stream_url: str = "https://cdn.example.com/track.enc.flac", + decryption_key: Optional[str] = "deadbeef1234", +) -> T2TunesStreamInfo: + return T2TunesStreamInfo( + asin=asin, + streamable=streamable, + codec=codec, + format=codec, + sample_rate=sample_rate, + stream_url=stream_url, + decryption_key=decryption_key, + title="Not Like Us", + artist="Kendrick Lamar", + album="GNX", + isrc="USRC12345678", + ) + + +def _search_items(n_tracks: int = 2, n_albums: int = 1): + from core.amazon_client import T2TunesSearchItem + items = [ + T2TunesSearchItem( + asin=f"B0TRACK{i}", + title=f"Track {i}", + artist_name="Kendrick Lamar", + item_type="MusicTrack", + album_name="GNX", + album_asin="B0ALBUM1", + duration_seconds=200 + i * 10, + isrc=f"USRC{i:08d}", + ) + for i in range(n_tracks) + ] + items += [ + T2TunesSearchItem( + asin=f"B0ALBUM{j}", + title=f"Album {j}", + artist_name="Kendrick Lamar", + item_type="MusicAlbum", + album_name=f"Album {j}", + album_asin=f"B0ALBUM{j}", + duration_seconds=0, + ) + for j in range(n_albums) + ] + return items + + +def _make_client(tmp_path: Path) -> AmazonDownloadClient: + with patch("core.amazon_download_client.config_manager") as cfg: + cfg.get.return_value = str(tmp_path) + with patch("core.amazon_client.config_manager") as cfg2: + cfg2.get.return_value = "" + client = AmazonDownloadClient(download_path=str(tmp_path)) + return client + + +def _fake_chunked_response(data: bytes, status_code: int = 200) -> MagicMock: + resp = MagicMock() + resp.status_code = status_code + resp.ok = status_code < 400 + resp.headers = {"content-length": str(len(data))} + + chunk_size = 4096 + chunks = [data[i : i + chunk_size] for i in range(0, len(data), chunk_size)] or [b""] + + def iter_content(chunk_size=None): + yield from chunks + + resp.iter_content = iter_content + if status_code >= 400: + from requests import HTTPError + resp.raise_for_status.side_effect = HTTPError(response=resp) + else: + resp.raise_for_status.return_value = None + return resp + + +# --------------------------------------------------------------------------- +# Codec / quality helpers +# --------------------------------------------------------------------------- + +class TestCodecHelpers: + def test_codec_key_lowercases(self): + assert _codec_key("FLAC") == "flac" + assert _codec_key("OGG-Vorbis") == "ogg_vorbis" + assert _codec_key("EAC3") == "eac3" + + def test_file_extension_known_codecs(self): + assert _file_extension("FLAC") == "flac" + assert _file_extension("ogg_vorbis") == "ogg" + assert _file_extension("opus") == "opus" + assert _file_extension("eac3") == "eac3" + assert _file_extension("mp4") == "m4a" + assert _file_extension("aac") == "m4a" + assert _file_extension("mp3") == "mp3" + + def test_file_extension_unknown_falls_back(self): + assert _file_extension("wtf_codec") == "bin" + + def test_quality_label_flac_lossless(self): + assert _quality_label("flac", 44100) == "Lossless" + assert _quality_label("FLAC", 48000) == "Lossless" + + def test_quality_label_flac_hires(self): + assert _quality_label("flac", 96000) == "Hi-Res" + assert _quality_label("flac", 192000) == "Hi-Res" + + def test_quality_label_lossy(self): + assert _quality_label("opus") == "Lossy" + assert _quality_label("eac3") == "Lossy" + assert _quality_label("mp3") == "Lossy" + + +# --------------------------------------------------------------------------- +# is_configured / check_connection +# --------------------------------------------------------------------------- + +class TestIsConfigured: + def test_always_true(self, tmp_path): + client = _make_client(tmp_path) + assert client.is_configured() is True + + +class TestCheckConnection: + def test_true_when_client_authenticated(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.is_authenticated.return_value = True + assert run(client.check_connection()) is True + + def test_false_when_client_down(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.is_authenticated.return_value = False + assert run(client.check_connection()) is False + + def test_false_on_exception(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.is_authenticated.side_effect = Exception("timeout") + assert run(client.check_connection()) is False + + +# --------------------------------------------------------------------------- +# search() +# --------------------------------------------------------------------------- + +class TestSearch: + def test_returns_track_results(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=2, n_albums=0) + client._client.preferred_codec = "flac" + + tracks, albums = run(client.search("Kendrick Lamar")) + + assert len(tracks) == 2 + assert len(albums) == 0 + assert all(isinstance(t, TrackResult) for t in tracks) + + def test_track_fields(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=1, n_albums=0) + client._client.preferred_codec = "flac" + + tracks, _ = run(client.search("Not Like Us")) + t = tracks[0] + + assert t.username == "amazon" + assert "B0TRACK0" in t.filename + assert "||" in t.filename + assert t.artist == "Kendrick Lamar" + assert t.title == "Track 0" + assert t.album == "GNX" + assert t.quality == "Lossless" + assert t.duration == 200_000 + + def test_track_source_metadata(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=1, n_albums=0) + client._client.preferred_codec = "flac" + + tracks, _ = run(client.search("test")) + meta = tracks[0]._source_metadata + + assert meta["asin"] == "B0TRACK0" + assert meta["album_asin"] == "B0ALBUM1" + assert "isrc" in meta + + def test_returns_album_results(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=0, n_albums=2) + client._client.preferred_codec = "flac" + + _, albums = run(client.search("GNX")) + + assert len(albums) == 2 + assert all(isinstance(a, AlbumResult) for a in albums) + + def test_album_deduplication(self, tmp_path): + from core.amazon_client import T2TunesSearchItem + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.preferred_codec = "flac" + # Two hits with same album_asin + dup = T2TunesSearchItem( + asin="B0ALBUM0", + title="GNX", + artist_name="Kendrick Lamar", + item_type="MusicAlbum", + album_asin="B0ALBUM0", + ) + client._client.search_raw.return_value = [dup, dup] + + _, albums = run(client.search("GNX")) + assert len(albums) == 1 + + def test_returns_empty_on_error(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.side_effect = AmazonClientError("fail") + + tracks, albums = run(client.search("anything")) + assert tracks == [] + assert albums == [] + + def test_soulseek_compat_fields(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=1, n_albums=0) + client._client.preferred_codec = "flac" + + tracks, _ = run(client.search("test")) + t = tracks[0] + + assert t.free_upload_slots == 999 + assert t.upload_speed == 999_999 + assert t.queue_length == 0 + assert t.size == 0 + + +# --------------------------------------------------------------------------- +# _unique_path +# --------------------------------------------------------------------------- + +class TestUniquePath: + def test_returns_original_when_no_conflict(self, tmp_path): + p = tmp_path / "track.flac" + result = AmazonDownloadClient._unique_path(p) + assert result == p + + def test_appends_counter_on_conflict(self, tmp_path): + p = tmp_path / "track.flac" + p.touch() + result = AmazonDownloadClient._unique_path(p) + assert result != p + assert "(1)" in result.name + + def test_increments_counter(self, tmp_path): + p = tmp_path / "track.flac" + p.touch() + (tmp_path / "track (1).flac").touch() + result = AmazonDownloadClient._unique_path(p) + assert "(2)" in result.name + + +# --------------------------------------------------------------------------- +# _record_to_status +# --------------------------------------------------------------------------- + +class TestRecordToStatus: + def test_fields_mapped(self): + rec = { + "id": "dl-001", + "filename": "B1||Artist - Title", + "state": "downloading", + "progress": 0.5, + "size": 10_000_000, + "transferred": 5_000_000, + "speed": 1_000_000, + "time_remaining": 5, + "file_path": "/tmp/track.flac", + } + status = AmazonDownloadClient._record_to_status(rec) + + assert status.id == "dl-001" + assert status.filename == "B1||Artist - Title" + assert status.username == "amazon" + assert status.state == "downloading" + assert status.progress == 0.5 + assert status.size == 10_000_000 + assert status.transferred == 5_000_000 + assert status.speed == 1_000_000 + assert status.time_remaining == 5 + assert status.file_path == "/tmp/track.flac" + + def test_defaults_for_missing_fields(self): + status = AmazonDownloadClient._record_to_status({}) + assert status.id == "" + assert status.state == "queued" + assert status.progress == 0.0 + assert status.size == 0 + assert status.transferred == 0 + assert status.speed == 0 + assert status.time_remaining is None + assert status.file_path is None + + +# --------------------------------------------------------------------------- +# _stream_to_file +# --------------------------------------------------------------------------- + +class TestStreamToFile: + def test_writes_file_and_returns_size(self, tmp_path): + client = _make_client(tmp_path) + data = b"X" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(data) + + out = tmp_path / "output.flac" + downloaded = client._stream_to_file("https://example.com/t.flac", out, "dl-001") + + assert downloaded == len(data) + assert out.exists() + assert out.read_bytes() == data + + def test_raises_on_http_error(self, tmp_path): + from requests import HTTPError + client = _make_client(tmp_path) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(b"", status_code=403) + + out = tmp_path / "output.flac" + with pytest.raises(HTTPError): + client._stream_to_file("https://example.com/t.flac", out, "dl-001") + + def test_respects_shutdown_check(self, tmp_path): + client = _make_client(tmp_path) + data = b"X" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(data) + client.shutdown_check = lambda: True # trigger immediately + + out = tmp_path / "output.flac" + with pytest.raises(RuntimeError, match="Shutdown"): + client._stream_to_file("https://example.com/t.flac", out, "dl-001") + assert not out.exists() + + def test_updates_engine_progress(self, tmp_path): + import itertools + client = _make_client(tmp_path) + data = b"X" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(data) + engine = MagicMock() + client._engine = engine + + out = tmp_path / "output.flac" + counter = itertools.count(0.0, 1.0) + with patch("core.amazon_download_client.time") as mock_time: + mock_time.monotonic.side_effect = lambda: next(counter) + client._stream_to_file("https://example.com/t.flac", out, "dl-001") + + assert engine.update_record.called + + +# --------------------------------------------------------------------------- +# _decrypt_with_ffmpeg +# --------------------------------------------------------------------------- + +class TestDecryptWithFfmpeg: + def test_calls_ffmpeg_with_key(self, tmp_path): + client = _make_client(tmp_path) + enc = tmp_path / "track.enc.flac" + enc.write_bytes(b"encrypted") + out = tmp_path / "track.flac" + + with patch("shutil.which", return_value="/usr/bin/ffmpeg"): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0, stderr=b"") + client._decrypt_with_ffmpeg(enc, out, "deadbeef1234") + + cmd = mock_run.call_args[0][0] + assert "ffmpeg" in cmd[0] + assert "-decryption_key" in cmd + assert "deadbeef1234" in cmd + assert str(enc) in cmd + assert str(out) in cmd + + def test_raises_on_ffmpeg_failure(self, tmp_path): + client = _make_client(tmp_path) + enc = tmp_path / "track.enc.flac" + enc.write_bytes(b"bad") + out = tmp_path / "track.flac" + + with patch("shutil.which", return_value="/usr/bin/ffmpeg"): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock( + returncode=1, stderr=b"Invalid data found" + ) + with pytest.raises(RuntimeError, match="FFmpeg decryption failed"): + client._decrypt_with_ffmpeg(enc, out, "deadbeef1234") + + def test_raises_when_ffmpeg_missing(self, tmp_path): + client = _make_client(tmp_path) + enc = tmp_path / "track.enc.flac" + out = tmp_path / "track.flac" + + with patch("shutil.which", return_value=None): + # Ensure tools/ffmpeg.exe also absent + with pytest.raises(RuntimeError, match="ffmpeg is required"): + client._decrypt_with_ffmpeg(enc, out, "deadbeef1234") + + def test_uses_tools_ffmpeg_when_not_on_path(self, tmp_path): + client = _make_client(tmp_path) + enc = tmp_path / "track.enc.flac" + enc.write_bytes(b"enc") + out = tmp_path / "track.flac" + + fake_ffmpeg = tmp_path / "ffmpeg.exe" + fake_ffmpeg.touch() + + with patch("shutil.which", return_value=None): + with patch( + "core.amazon_download_client.Path.__file__", + create=True, + ): + import os as _os + is_nt = _os.name == "nt" + ffmpeg_name = "ffmpeg.exe" if is_nt else "ffmpeg" + tools_dir = ROOT / "tools" + tools_ffmpeg = tools_dir / ffmpeg_name + if tools_ffmpeg.exists(): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0, stderr=b"") + client._decrypt_with_ffmpeg(enc, out, "aabbcc") + assert str(tools_ffmpeg) in mock_run.call_args[0][0][0] + + +# --------------------------------------------------------------------------- +# _download_sync — integration of stream + decrypt +# --------------------------------------------------------------------------- + +class TestDownloadSync: + def _setup(self, tmp_path: Path, decryption_key: Optional[str] = "deadbeef"): + client = _make_client(tmp_path) + stream = _stream_info(decryption_key=decryption_key) + client._client = MagicMock() + client._client.media_from_asin.return_value = [stream] + client._client.preferred_codec = "flac" + + audio_data = b"A" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(audio_data) + return client, audio_data + + def test_returns_output_path_on_success(self, tmp_path): + client, audio_data = self._setup(tmp_path) + + with patch("shutil.which", return_value="/usr/bin/ffmpeg"): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0, stderr=b"") + # simulate ffmpeg writing output file + def _ffmpeg_side_effect(cmd, capture_output=False): + # Write dummy decrypted data + out_path = Path(cmd[-1]) + out_path.write_bytes(audio_data) + return MagicMock(returncode=0, stderr=b"") + + mock_run.side_effect = _ffmpeg_side_effect + result = client._download_sync("dl-001", "B09XYZ1234", "Kendrick Lamar - Not Like Us") + + assert result is not None + assert Path(result).exists() + assert Path(result).suffix == ".flac" + + def test_tries_next_codec_when_stream_unavailable(self, tmp_path): + client = _make_client(tmp_path) + # FLAC: no streamable results; Opus: success + flac_stream = _stream_info(streamable=False, codec="FLAC") + opus_stream = _stream_info(streamable=True, codec="OPUS", + stream_url="https://cdn.example.com/t.opus", + decryption_key=None) + client._client = MagicMock() + client._client.preferred_codec = "flac" + + def _media(asin, codec): + if codec == "flac": + return [flac_stream] + if codec == "opus": + return [opus_stream] + return [] + + client._client.media_from_asin.side_effect = _media + + audio_data = b"B" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(audio_data) + + result = client._download_sync("dl-001", "B09XYZ1234", "Kendrick - Track") + + assert result is not None + assert ".opus" in result + + def test_returns_none_when_file_too_small(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.preferred_codec = "flac" + client._client.media_from_asin.return_value = [ + _stream_info(decryption_key=None) + ] + tiny_data = b"X" * 100 # way below MIN_AUDIO_BYTES + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(tiny_data) + + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Title") + assert result is None + + def test_tries_next_codec_when_media_fails(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.preferred_codec = "flac" + + call_count = {"n": 0} + + def _media(asin, codec): + call_count["n"] += 1 + if call_count["n"] == 1: + raise AmazonClientError("quota exceeded") + stream = _stream_info(codec="OPUS", decryption_key=None) + return [stream] + + client._client.media_from_asin.side_effect = _media + + audio_data = b"C" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(audio_data) + + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + assert result is not None + + def test_decryption_failure_tries_next_codec(self, tmp_path): + client, audio_data = self._setup(tmp_path, decryption_key="badkey") + + with patch("shutil.which", return_value="/usr/bin/ffmpeg"): + with patch("subprocess.run") as mock_run: + # First codec (flac) decryption fails; no more codecs succeed + mock_run.return_value = MagicMock(returncode=1, stderr=b"decrypt error") + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + + # All codecs should fail since we return the same bad result for all + assert result is None + + def test_updates_engine_state(self, tmp_path): + client, audio_data = self._setup(tmp_path, decryption_key=None) + engine = MagicMock() + client._engine = engine + + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + + update_calls = engine.update_record.call_args_list + states = [c[0][2].get("state") for c in update_calls if "state" in c[0][2]] + assert "downloading" in states + + def test_final_size_update_syncs_transferred_to_size(self, tmp_path): + """size == transferred after success so monitor bytes-incomplete guard doesn't block.""" + client, audio_data = self._setup(tmp_path, decryption_key=None) + engine = MagicMock() + client._engine = engine + + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + + assert result is not None + # Last update must set size == transferred (both equal the output file size) + final_calls = [ + c[0][2] for c in engine.update_record.call_args_list + if 'size' in c[0][2] and 'transferred' in c[0][2] + and c[0][2]['size'] == c[0][2]['transferred'] + and c[0][2]['size'] > 0 + ] + assert final_calls, "Expected a final engine update with size == transferred > 0" + + def test_clear_stream_skips_ffmpeg(self, tmp_path): + client, audio_data = self._setup(tmp_path, decryption_key=None) + + with patch("subprocess.run") as mock_run: + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + + mock_run.assert_not_called() + assert result is not None + + def test_safe_filename_sanitisation(self, tmp_path): + client, audio_data = self._setup(tmp_path, decryption_key=None) + result = client._download_sync( + "dl-001", "B09XYZ1234", "Björk / Sigur Rós: Hvarf<>Heim" + ) + assert result is not None + # Path must not contain illegal filesystem chars + path = Path(result) + assert "/" not in path.name + assert "<" not in path.name + assert ">" not in path.name + + +# --------------------------------------------------------------------------- +# download() — async dispatch +# --------------------------------------------------------------------------- + +class TestDownloadDispatch: + def test_raises_without_engine(self, tmp_path): + client = _make_client(tmp_path) + with pytest.raises(RuntimeError, match="_engine"): + run(client.download("amazon", "B09XYZ1234||Artist - Title")) + + def test_returns_none_on_bad_filename(self, tmp_path): + client = _make_client(tmp_path) + client._engine = MagicMock() + result = run(client.download("amazon", "no-pipe-delimiter-here")) + assert result is None + + def test_dispatches_to_engine_worker(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.worker.dispatch.return_value = "dl-abc123" + client._engine = engine + + result = run(client.download("amazon", "B09XYZ1234||Kendrick Lamar - Not Like Us")) + + assert result == "dl-abc123" + dispatch_call = engine.worker.dispatch.call_args + assert dispatch_call[1]["source_name"] == "amazon" + assert dispatch_call[1]["target_id"] == "B09XYZ1234" + assert dispatch_call[1]["display_name"] == "Kendrick Lamar - Not Like Us" + assert dispatch_call[1]["impl_callable"] == client._download_sync + + def test_strips_whitespace_from_asin_and_name(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.worker.dispatch.return_value = "dl-xyz" + client._engine = engine + + run(client.download("amazon", " B09XYZ1234 || Artist - Title ")) + + call = engine.worker.dispatch.call_args + assert call[1]["target_id"] == "B09XYZ1234" + assert call[1]["display_name"] == "Artist - Title" + + def test_set_engine_wires_engine(self, tmp_path): + client = _make_client(tmp_path) + assert client._engine is None + engine = MagicMock() + client.set_engine(engine) + assert client._engine is engine + + def test_set_shutdown_check_wires_callback(self, tmp_path): + client = _make_client(tmp_path) + assert client.shutdown_check is None + check = lambda: False + client.set_shutdown_check(check) + assert client.shutdown_check is check + + def test_set_engine_allows_download_dispatch(self, tmp_path): + """set_engine() must unblock download() — the live failure mode.""" + client = _make_client(tmp_path) + engine = MagicMock() + engine.worker.dispatch.return_value = "dl-wired" + client.set_engine(engine) + result = run(client.download("amazon", "B09XYZ1234||Artist - Title")) + assert result == "dl-wired" + + +# --------------------------------------------------------------------------- +# Status interface +# --------------------------------------------------------------------------- + +class TestStatusInterface: + def test_get_all_downloads_empty_without_engine(self, tmp_path): + client = _make_client(tmp_path) + result = run(client.get_all_downloads()) + assert result == [] + + def test_get_all_downloads_converts_records(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.iter_records_for_source.return_value = iter([{ + "id": "dl-001", + "filename": "B1||A - T", + "state": "complete", + "progress": 1.0, + "size": 5_000_000, + "transferred": 5_000_000, + "speed": 0, + }]) + client._engine = engine + + statuses = run(client.get_all_downloads()) + assert len(statuses) == 1 + assert statuses[0].id == "dl-001" + assert statuses[0].state == "complete" + engine.iter_records_for_source.assert_called_once_with('amazon') + + def test_get_download_status_returns_none_without_engine(self, tmp_path): + client = _make_client(tmp_path) + assert run(client.get_download_status("dl-001")) is None + + def test_get_download_status_hit(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.get_record.return_value = { + "id": "dl-001", + "filename": "B1||A - T", + "state": "downloading", + "progress": 0.7, + "size": 10_000_000, + "transferred": 7_000_000, + "speed": 500_000, + } + client._engine = engine + + status = run(client.get_download_status("dl-001")) + assert status is not None + assert status.id == "dl-001" + assert status.state == "downloading" + assert status.progress == 0.7 + engine.get_record.assert_called_once_with('amazon', 'dl-001') + + def test_get_download_status_miss(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.get_record.return_value = None + client._engine = engine + + assert run(client.get_download_status("nonexistent")) is None + + def test_cancel_returns_false_without_engine(self, tmp_path): + client = _make_client(tmp_path) + assert run(client.cancel_download("dl-001")) is False + + def test_cancel_delegates_to_engine(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.get_record.return_value = {'id': 'dl-001', 'state': 'downloading'} + client._engine = engine + + result = run(client.cancel_download("dl-001", remove=True)) + assert result is True + engine.update_record.assert_called_once_with('amazon', 'dl-001', {'state': 'Cancelled'}) + engine.remove_record.assert_called_once_with('amazon', 'dl-001') + + def test_cancel_returns_false_when_record_not_found(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.get_record.return_value = None + client._engine = engine + + result = run(client.cancel_download("nonexistent")) + assert result is False + engine.update_record.assert_not_called() + + def test_clear_completed_returns_true_without_engine(self, tmp_path): + client = _make_client(tmp_path) + assert run(client.clear_all_completed_downloads()) is True + + def test_clear_completed_removes_terminal_records(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.iter_records_for_source.return_value = iter([ + {'id': 'dl-001', 'state': 'Completed, Succeeded'}, + {'id': 'dl-002', 'state': 'InProgress, Downloading'}, + {'id': 'dl-003', 'state': 'Errored'}, + ]) + client._engine = engine + + result = run(client.clear_all_completed_downloads()) + assert result is True + # Only terminal records removed + removed_ids = [c[0][1] for c in engine.remove_record.call_args_list] + assert 'dl-001' in removed_ids + assert 'dl-003' in removed_ids + assert 'dl-002' not in removed_ids + + def test_status_methods_no_records(self, tmp_path): + """Engine with no Amazon records returns empty/None gracefully.""" + client = _make_client(tmp_path) + engine = MagicMock() + engine.iter_records_for_source.return_value = iter([]) + engine.get_record.return_value = None + client._engine = engine + + assert run(client.get_all_downloads()) == [] + assert run(client.get_download_status("dl-001")) is None + assert run(client.cancel_download("dl-001")) is False + assert run(client.clear_all_completed_downloads()) is True diff --git a/tests/tools/test_amazon_worker_schema.py b/tests/tools/test_amazon_worker_schema.py new file mode 100644 index 00000000..3e6a5125 --- /dev/null +++ b/tests/tools/test_amazon_worker_schema.py @@ -0,0 +1,63 @@ +import sqlite3 + +from core.amazon_worker import AmazonWorker + + +class _LegacyDatabase: + def __init__(self, path): + self.path = str(path) + with sqlite3.connect(self.path) as conn: + conn.executescript( + """ + CREATE TABLE artists ( + id INTEGER PRIMARY KEY, + name TEXT + ); + CREATE TABLE albums ( + id INTEGER PRIMARY KEY, + title TEXT, + artist_id INTEGER + ); + CREATE TABLE tracks ( + id INTEGER PRIMARY KEY, + title TEXT, + artist_id INTEGER + ); + INSERT INTO artists (id, name) VALUES (1, 'Artist A'); + INSERT INTO albums (id, title, artist_id) VALUES (10, 'Album A', 1); + INSERT INTO tracks (id, title, artist_id) VALUES (100, 'Track A', 1); + """ + ) + + def _get_connection(self): + return sqlite3.connect(self.path) + + +def _columns(db_path, table): + with sqlite3.connect(str(db_path)) as conn: + return {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} + + +def test_amazon_worker_self_heals_legacy_schema_before_selecting_next_item(tmp_path): + db_path = tmp_path / "legacy.db" + db = _LegacyDatabase(db_path) + worker = AmazonWorker(db) + + item = worker._get_next_item() + + assert item == {"type": "artist", "id": 1, "name": "Artist A"} + for table in ("artists", "albums", "tracks"): + assert {"amazon_id", "amazon_match_status", "amazon_last_attempted"} <= _columns(db_path, table) + + +def test_amazon_worker_stats_self_heal_legacy_schema(tmp_path): + db_path = tmp_path / "legacy.db" + db = _LegacyDatabase(db_path) + worker = AmazonWorker(db) + + assert worker._count_pending_items() == 3 + assert worker._get_progress_breakdown() == { + "artists": {"matched": 0, "total": 1, "percent": 0}, + "albums": {"matched": 0, "total": 1, "percent": 0}, + "tracks": {"matched": 0, "total": 1, "percent": 0}, + } diff --git a/tests/tools/test_t2tunes_probe.py b/tests/tools/test_t2tunes_probe.py new file mode 100644 index 00000000..9856da32 --- /dev/null +++ b/tests/tools/test_t2tunes_probe.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import Mock + +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) + +from t2tunes_probe import T2TunesClient, T2TunesError # noqa: E402 + + +class _Response: + def __init__(self, payload=None, *, status_code=200, text="", headers=None, url="https://example.test/x"): + self._payload = payload + self.status_code = status_code + self.text = text + self.headers = headers or {} + self.url = url + self.ok = 200 <= status_code < 400 + + def json(self): + if isinstance(self._payload, Exception): + raise self._payload + return self._payload + + def raise_for_status(self): + if not self.ok: + raise RuntimeError(f"{self.status_code} error") + + +class T2TunesProbeTests(unittest.TestCase): + def test_status_uses_api_status_endpoint(self): + session = Mock() + session.get.return_value = _Response({"amazonMusic": "up"}) + client = T2TunesClient("https://t2.example", session=session) + + self.assertTrue(client.amazon_music_is_up()) + session.get.assert_called_once() + self.assertEqual(session.get.call_args.args[0], "https://t2.example/api/status") + + def test_search_flattens_nested_hits(self): + session = Mock() + session.get.return_value = _Response({ + "results": [{ + "hits": [{ + "document": { + "__type": "Track", + "asin": "B001", + "title": "Song", + "artistName": "Artist", + "albumName": "Album", + "duration": 123, + "isrc": "USABC123", + } + }] + }] + }) + client = T2TunesClient("https://t2.example", session=session) + + items = client.search("artist song") + + self.assertEqual(len(items), 1) + self.assertEqual(items[0].asin, "B001") + self.assertTrue(items[0].is_track) + self.assertEqual(items[0].duration_seconds, 123) + self.assertEqual(session.get.call_args.kwargs["params"]["types"], "track,album") + + def test_media_handles_streamable_typo_and_decryption_flag(self): + session = Mock() + session.get.return_value = _Response([{ + "asin": "B001", + "stremeable": True, + "decryptionKey": "abc123", + "tags": { + "title": "Song", + "artist": "Artist", + "album": "Album", + "isrc": "USABC123", + }, + "streamInfo": { + "format": "flac", + "codec": "flac", + "sampleRate": 48000, + "streamUrl": "https://cdn.example/song.flac", + }, + }]) + client = T2TunesClient("https://t2.example", session=session) + + streams = client.media_from_asin("B001") + + self.assertEqual(len(streams), 1) + self.assertTrue(streams[0].streamable) + self.assertTrue(streams[0].has_decryption_key) + self.assertEqual(streams[0].stream_url, "https://cdn.example/song.flac") + + def test_non_json_response_raises_probe_error(self): + session = Mock() + session.get.return_value = _Response(ValueError("not json"), text="Service Unavailable") + client = T2TunesClient("https://t2.example", session=session) + + with self.assertRaises(T2TunesError): + client.status() + + def test_probe_stream_falls_back_to_range_get_when_head_is_blocked(self): + session = Mock() + session.head.return_value = _Response(status_code=405) + session.get.return_value = _Response( + status_code=206, + headers={"content-type": "audio/flac", "content-length": "1"}, + url="https://cdn.example/song.flac", + ) + client = T2TunesClient("https://t2.example", session=session) + + result = client.probe_stream("https://cdn.example/song.flac") + + self.assertTrue(result["ok"]) + self.assertEqual(result["method"], "GET range") + self.assertEqual(result["content_type"], "audio/flac") + self.assertEqual(session.get.call_args.kwargs["headers"], {"Range": "bytes=0-0"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/watchlist/test_musicbrainz_watchlist_ids.py b/tests/watchlist/test_musicbrainz_watchlist_ids.py new file mode 100644 index 00000000..a4092d94 --- /dev/null +++ b/tests/watchlist/test_musicbrainz_watchlist_ids.py @@ -0,0 +1,81 @@ +from database.music_database import MusicDatabase + + +def test_watchlist_artist_can_store_musicbrainz_match(tmp_path): + db = MusicDatabase(str(tmp_path / "music.db")) + + assert db.add_artist_to_watchlist( + "mb-artist-1", + "MusicBrainz Artist", + profile_id=1, + source="musicbrainz", + ) + + artists = db.get_watchlist_artists(profile_id=1) + + assert len(artists) == 1 + assert artists[0].artist_name == "MusicBrainz Artist" + assert artists[0].musicbrainz_artist_id == "mb-artist-1" + assert artists[0].spotify_artist_id is None + + +def test_watchlist_musicbrainz_match_can_be_added_to_existing_artist(tmp_path): + db = MusicDatabase(str(tmp_path / "music.db")) + + assert db.add_artist_to_watchlist("sp-artist-1", "Linked Artist", profile_id=1, source="spotify") + assert db.add_artist_to_watchlist("mb-artist-1", "Linked Artist", profile_id=1, source="musicbrainz") + + artists = db.get_watchlist_artists(profile_id=1) + + assert len(artists) == 1 + assert artists[0].spotify_artist_id == "sp-artist-1" + assert artists[0].musicbrainz_artist_id == "mb-artist-1" + + +def test_watchlist_musicbrainz_match_supports_presence_and_removal(tmp_path): + db = MusicDatabase(str(tmp_path / "music.db")) + db.add_artist_to_watchlist("sp-artist-1", "Removable Artist", profile_id=1, source="spotify") + artist = db.get_watchlist_artists(profile_id=1)[0] + + assert db.update_watchlist_musicbrainz_id(artist.id, "mb-artist-1") + assert db.is_artist_in_watchlist("mb-artist-1", profile_id=1) + assert db.remove_artist_from_watchlist("mb-artist-1", profile_id=1) + assert db.get_watchlist_artists(profile_id=1) == [] + + +def test_watchlist_musicbrainz_match_backfills_from_library_by_name(tmp_path): + db = MusicDatabase(str(tmp_path / "music.db")) + db.add_artist_to_watchlist("sp-artist-1", "Library Matched Artist", profile_id=1, source="spotify") + with db._get_connection() as conn: + conn.execute( + """ + INSERT INTO artists (id, name, musicbrainz_id) + VALUES (?, ?, ?) + """, + ("library-artist-1", "Library Matched Artist", "mb-library-1"), + ) + conn.commit() + + assert db.backfill_watchlist_musicbrainz_ids_from_library(profile_id=1) == 1 + + artist = db.get_watchlist_artists(profile_id=1)[0] + assert artist.musicbrainz_artist_id == "mb-library-1" + + +def test_watchlist_musicbrainz_match_backfills_from_library_by_linked_id(tmp_path): + db = MusicDatabase(str(tmp_path / "music.db")) + db.add_artist_to_watchlist("sp-artist-1", "Different Watchlist Name", profile_id=1, source="spotify") + with db._get_connection() as conn: + conn.execute( + """ + INSERT INTO artists (id, name, spotify_artist_id, musicbrainz_id) + VALUES (?, ?, ?, ?) + """, + ("library-artist-1", "Canonical Library Name", "sp-artist-1", "mb-library-1"), + ) + conn.commit() + + assert db.backfill_watchlist_musicbrainz_ids_from_library(profile_id=1) == 1 + + artist = db.get_watchlist_artists(profile_id=1)[0] + assert artist.musicbrainz_artist_id == "mb-library-1" diff --git a/tests/webui/test_assets.py b/tests/webui/test_assets.py index 21660899..8c6a921d 100644 --- a/tests/webui/test_assets.py +++ b/tests/webui/test_assets.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import os +from pathlib import Path import time import pytest @@ -94,3 +95,18 @@ def test_load_webui_vite_manifest_reloads_when_file_changes(tmp_path): second = load_webui_vite_manifest(manifest_path) assert second["src/app/main.tsx"]["file"] == "assets/two.js" + + +def test_static_ui_uses_existing_album_placeholder_asset(): + repo_root = Path(__file__).resolve().parents[2] + static_dir = repo_root / "webui" / "static" + + assert (static_dir / "placeholder-album.png").exists() + assert not (static_dir / "placeholder.png").exists() + + stale_refs = [] + for path in static_dir.glob("*.js"): + if "/static/placeholder.png" in path.read_text(encoding="utf-8"): + stale_refs.append(path.name) + + assert stale_refs == [] diff --git a/tests/wishlist/test_processing.py b/tests/wishlist/test_processing.py index d5fb2bff..709b3cc6 100644 --- a/tests/wishlist/test_processing.py +++ b/tests/wishlist/test_processing.py @@ -225,6 +225,38 @@ def test_automatic_wishlist_cleanup_after_db_update_removes_library_matches(): assert wishlist_service.removed == [("sp-1", True, None, 1)] +def test_automatic_wishlist_cleanup_after_db_update_removes_manual_matches(monkeypatch): + wishlist_service = _CleanupWishlistService( + [ + { + "name": "Manual Song", + "artists": [{"name": "Artist A"}], + "spotify_track_id": "sp-manual", + "id": "sp-manual", + "provider": "spotify", + "album": {"name": "Album A"}, + }, + ] + ) + music_db = _CleanupMusicDatabase() + monkeypatch.setattr( + "core.library.manual_library_match.get_match_for_track", + lambda *_args, **_kwargs: {"id": 1, "library_track_id": 42}, + ) + + removed = processing.automatic_wishlist_cleanup_after_db_update( + wishlist_service=wishlist_service, + profiles_database=_CleanupProfilesDatabase(), + music_database=music_db, + active_server="navidrome", + logger=_FakeLogger(), + ) + + assert removed == 1 + assert wishlist_service.removed == [("sp-manual", True, None, 1)] + assert music_db.track_checks == [] + + class _CleanupProfilesDatabase: def get_all_profiles(self): return [{"id": 1}] diff --git a/tools/diagnose_itunes_discover.py b/tools/diagnose_itunes_discover.py index ff047c26..8aa9c821 100644 --- a/tools/diagnose_itunes_discover.py +++ b/tools/diagnose_itunes_discover.py @@ -63,9 +63,17 @@ def diagnose_itunes_discover(): """) with_both = cursor.fetchone()['count'] + with_musicbrainz = 0 + try: + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_musicbrainz_id IS NOT NULL") + with_musicbrainz = cursor.fetchone()['count'] + except Exception as exc: + logger.debug("similar_artist_musicbrainz_id column is unavailable: %s", exc) + logger.info(f" Total similar artists: {total}") logger.info(f" With iTunes ID: {with_itunes} ({100 * with_itunes / total:.1f}%)" if total > 0 else " With iTunes ID: 0") logger.info(f" With Spotify ID: {with_spotify} ({100 * with_spotify / total:.1f}%)" if total > 0 else " With Spotify ID: 0") + logger.info(f" With MusicBrainz ID: {with_musicbrainz} ({100 * with_musicbrainz / total:.1f}%)" if total > 0 else " With MusicBrainz ID: 0") logger.info(f" With BOTH IDs: {with_both} ({100 * with_both / total:.1f}%)" if total > 0 else " With BOTH IDs: 0") if with_itunes == 0 and total > 0: diff --git a/tools/t2tunes_media_plan.py b/tools/t2tunes_media_plan.py new file mode 100644 index 00000000..531e2f89 --- /dev/null +++ b/tools/t2tunes_media_plan.py @@ -0,0 +1,199 @@ +"""Build a T2Tunes media plan for a track without downloading media. + +This runner calls ``tools.t2tunes_probe.T2TunesClient`` and validates the +parts of a future download source that are safe to exercise live: + +- search resolution +- album metadata lookup +- cover URL discovery + HEAD probe +- per-codec media lookup +- stream URL HEAD/range probe + +It writes a JSON plan that a developer can inspect while building an +integration. It intentionally does not download or decrypt protected media. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +TOOL_DIR = Path(__file__).resolve().parent +if str(TOOL_DIR) not in sys.path: + sys.path.insert(0, str(TOOL_DIR)) + +from t2tunes_probe import T2TunesClient, T2TunesSearchItem # noqa: E402 + + +def _slug(value: str) -> str: + cleaned = "".join(ch.lower() if ch.isalnum() else "-" for ch in value) + parts = [part for part in cleaned.split("-") if part] + return "-".join(parts[:12]) or "t2tunes" + + +def _item_score(item: T2TunesSearchItem, query: str, explicit: Optional[bool]) -> int: + text = f"{item.artist_name} {item.title} {item.album_name}".lower() + score = 0 + for token in query.lower().split(): + if token in text: + score += 10 + if item.is_track: + score += 20 + if explicit is True and "explicit" in text: + score += 50 + if explicit is False and "clean" in text: + score += 50 + if explicit is True and "clean" in text: + score -= 40 + if explicit is False and "explicit" in text: + score -= 40 + return score + + +def _choose_result( + items: Iterable[T2TunesSearchItem], + *, + query: str, + asin: Optional[str], + explicit: Optional[bool], +) -> Optional[T2TunesSearchItem]: + items = list(items) + if asin: + return next((item for item in items if item.asin == asin or item.album_asin == asin), None) + if not items: + return None + return max(items, key=lambda item: _item_score(item, query, explicit)) + + +def _first_album(metadata: Dict[str, Any]) -> Dict[str, Any]: + albums = metadata.get("albumList") + if isinstance(albums, list) and albums: + album = albums[0] + return album if isinstance(album, dict) else {} + return {} + + +def _cover_url_from_metadata(metadata: Dict[str, Any]) -> str: + album = _first_album(metadata) + image = album.get("image") + return image if isinstance(image, str) else "" + + +def _head_url(client: T2TunesClient, url: str) -> Dict[str, Any]: + if not url: + return {"ok": False, "reason": "missing url"} + return client.probe_stream(url) + + +def build_plan( + *, + query: str, + base_url: str, + country: str, + codecs: List[str], + asin: Optional[str], + explicit: Optional[bool], + timeout: int, + probe_urls: bool, +) -> Dict[str, Any]: + client = T2TunesClient(base_url, country=country, timeout=timeout) + status = client.status() + search_items = client.search(query) + selected = _choose_result(search_items, query=query, asin=asin, explicit=explicit) + + plan: Dict[str, Any] = { + "base_url": base_url, + "country": country, + "query": query, + "status": status, + "result_count": len(search_items), + "selected": selected.__dict__ if selected else None, + "album_metadata": None, + "cover": None, + "formats": [], + "notes": [ + "This plan validates API behavior only.", + "It intentionally does not download or decrypt protected media.", + ], + } + + if not selected: + return plan + + album_asin = selected.album_asin or selected.asin + metadata = client.album_metadata(album_asin) + cover_url = _cover_url_from_metadata(metadata) + plan["album_metadata"] = { + "asin": album_asin, + "title": _first_album(metadata).get("title"), + "track_count": _first_album(metadata).get("trackCount"), + "image": cover_url, + "label": _first_album(metadata).get("label"), + } + plan["cover"] = { + "url": cover_url, + "probe": _head_url(client, cover_url) if probe_urls else None, + } + + for codec in codecs: + format_client = T2TunesClient(base_url, country=country, codec=codec, timeout=timeout) + streams = format_client.media_from_asin(selected.asin) + entries = [] + for stream in streams: + entry = stream.__dict__.copy() + entry["stream_probe"] = _head_url(format_client, stream.stream_url) if probe_urls else None + entries.append(entry) + plan["formats"].append({ + "requested_codec": codec, + "stream_count": len(entries), + "streams": entries, + }) + + return plan + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create a safe T2Tunes media plan for a track.") + parser.add_argument("query", help="Search query, e.g. 'Kendrick Lamar Not Like Us'") + parser.add_argument("--base-url", default="https://t2tunes.site") + parser.add_argument("--country", default="US") + parser.add_argument("--codec", action="append", dest="codecs", choices=("flac", "opus", "eac3")) + parser.add_argument("--asin", help="Prefer a specific track or album ASIN from search results") + group = parser.add_mutually_exclusive_group() + group.add_argument("--explicit", action="store_true", help="Prefer explicit search results") + group.add_argument("--clean", action="store_true", help="Prefer clean search results") + parser.add_argument("--timeout", type=int, default=30) + parser.add_argument("--no-probe", action="store_true", help="Do not HEAD-probe cover/stream URLs") + parser.add_argument( + "--output", + help="Optional JSON output path. Defaults to ./testTEST/-plan.json", + ) + args = parser.parse_args() + + explicit = True if args.explicit else False if args.clean else None + plan = build_plan( + query=args.query, + base_url=args.base_url, + country=args.country, + codecs=args.codecs or ["flac", "opus", "eac3"], + asin=args.asin, + explicit=explicit, + timeout=args.timeout, + probe_urls=not args.no_probe, + ) + + text = json.dumps(plan, indent=2, sort_keys=True) + output_path = Path(args.output) if args.output else Path.cwd() / "testTEST" / f"{_slug(args.query)}-plan.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(text + "\n", encoding="utf-8") + plan["_written_to"] = str(output_path) + text = json.dumps(plan, indent=2, sort_keys=True) + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/t2tunes_probe.py b/tools/t2tunes_probe.py new file mode 100644 index 00000000..95e18c09 --- /dev/null +++ b/tools/t2tunes_probe.py @@ -0,0 +1,369 @@ +"""Standalone T2Tunes/TripleTriple API probe. + +This is intentionally not wired into SoulSync's download source registry. +It validates the API surface Tubifarry targets: + + /api/status + /api/amazon-music/search + /api/amazon-music/metadata + /api/amazon-music/media-from-asin + +The probe can inspect returned stream metadata and optionally issue a HEAD +request against the stream URL. It does not download or decrypt audio. +""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Optional +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, urljoin +from urllib.request import Request, urlopen + + +DEFAULT_BASE_URL = "https://t2tunes.site" +DEFAULT_TIMEOUT_SECONDS = 30 + + +class T2TunesError(RuntimeError): + """Raised when the T2Tunes API returns an invalid or failed response.""" + + +class _HttpResponse: + def __init__(self, *, body: bytes, status_code: int, headers: Dict[str, str], url: str) -> None: + self.body = body + self.status_code = status_code + self.headers = headers + self.url = url + self.ok = 200 <= status_code < 400 + self.text = body.decode("utf-8", errors="replace") + + def json(self) -> Any: + return json.loads(self.text) + + def raise_for_status(self) -> None: + if not self.ok: + raise T2TunesError(f"HTTP {self.status_code} for {self.url}") + + +class _UrllibSession: + def get( + self, + url: str, + *, + params: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + timeout: int = DEFAULT_TIMEOUT_SECONDS, + allow_redirects: bool = True, # kept for test/session compatibility + stream: bool = False, # kept for test/session compatibility + ) -> _HttpResponse: + del allow_redirects, stream + if params: + separator = "&" if "?" in url else "?" + url = f"{url}{separator}{urlencode(params)}" + return self._request("GET", url, headers=headers, timeout=timeout) + + def head( + self, + url: str, + *, + headers: Optional[Dict[str, str]] = None, + timeout: int = DEFAULT_TIMEOUT_SECONDS, + allow_redirects: bool = True, # kept for test/session compatibility + ) -> _HttpResponse: + del allow_redirects + return self._request("HEAD", url, headers=headers, timeout=timeout) + + def _request( + self, + method: str, + url: str, + *, + headers: Optional[Dict[str, str]] = None, + timeout: int = DEFAULT_TIMEOUT_SECONDS, + ) -> _HttpResponse: + request = Request(url, headers=headers or {}, method=method) + try: + with urlopen(request, timeout=timeout) as response: + return _HttpResponse( + body=response.read(), + status_code=response.status, + headers={k.lower(): v for k, v in response.headers.items()}, + url=response.url, + ) + except HTTPError as exc: + body = exc.read() if hasattr(exc, "read") else b"" + return _HttpResponse( + body=body, + status_code=exc.code, + headers={k.lower(): v for k, v in exc.headers.items()} if exc.headers else {}, + url=exc.url, + ) + except URLError as exc: + raise T2TunesError(f"Network error for {url}: {exc}") from exc + + +@dataclass(frozen=True) +class T2TunesSearchItem: + asin: str + title: str + artist_name: str + item_type: str + album_name: str = "" + album_asin: str = "" + duration_seconds: int = 0 + isrc: str = "" + + @property + def is_album(self) -> bool: + return "album" in self.item_type.lower() + + @property + def is_track(self) -> bool: + return "track" in self.item_type.lower() + + +@dataclass(frozen=True) +class T2TunesStreamInfo: + asin: str + streamable: bool + codec: str + format: str + sample_rate: Optional[int] + stream_url: str + has_decryption_key: bool + title: str = "" + artist: str = "" + album: str = "" + isrc: str = "" + + +class T2TunesClient: + def __init__( + self, + base_url: str = DEFAULT_BASE_URL, + *, + country: str = "US", + codec: str = "flac", + timeout: int = DEFAULT_TIMEOUT_SECONDS, + session: Optional[Any] = None, + ) -> None: + self.base_url = base_url.rstrip("/") + self.country = country.upper() + self.codec = codec.lower() + self.timeout = timeout + self.session = session or _UrllibSession() + + def status(self) -> Dict[str, Any]: + return self._get_json("/api/status") + + def amazon_music_is_up(self) -> bool: + status = self.status() + return str(status.get("amazonMusic", "")).lower() == "up" + + def search(self, query: str, *, types: str = "track,album") -> List[T2TunesSearchItem]: + data = self._get_json( + "/api/amazon-music/search", + params={ + "query": query, + "types": types, + "country": self.country, + }, + ) + return list(_iter_search_items(data)) + + def album_metadata(self, asin: str) -> Dict[str, Any]: + return self._get_json( + "/api/amazon-music/metadata", + params={ + "asin": asin, + "country": self.country, + }, + ) + + def media_from_asin(self, asin: str) -> List[T2TunesStreamInfo]: + data = self._get_json( + "/api/amazon-music/media-from-asin", + params={ + "asin": asin, + "country": self.country, + "codec": self.codec, + }, + ) + if isinstance(data, list): + return [_media_response_to_stream_info(item) for item in data if isinstance(item, dict)] + if isinstance(data, dict): + return [_media_response_to_stream_info(data)] + raise T2TunesError(f"Unexpected media response type: {type(data).__name__}") + + def probe_stream(self, stream_url: str) -> Dict[str, Any]: + """Probe a stream URL without downloading audio bytes.""" + try: + response = self.session.head(stream_url, timeout=self.timeout, allow_redirects=True) + method = "HEAD" + if response.status_code in (405, 403): + response = self.session.get( + stream_url, + timeout=self.timeout, + allow_redirects=True, + stream=True, + headers={"Range": "bytes=0-0"}, + ) + method = "GET range" + return { + "ok": response.ok, + "method": method, + "status_code": response.status_code, + "content_type": response.headers.get("content-type", ""), + "content_length": response.headers.get("content-length", ""), + "accept_ranges": response.headers.get("accept-ranges", ""), + "final_url": response.url, + } + except Exception as exc: + return { + "ok": False, + "error": str(exc), + } + + def _get_json(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any: + url = urljoin(f"{self.base_url}/", path.lstrip("/")) + headers = { + "Accept": "application/json", + "Referer": self.base_url, + "User-Agent": "SoulSync-T2Tunes-Probe/0.1", + } + try: + response = self.session.get(url, params=params, headers=headers, timeout=self.timeout) + response.raise_for_status() + except Exception as exc: + raise T2TunesError(f"Request failed for {url}: {exc}") from exc + + try: + return response.json() + except ValueError as exc: + preview = response.text[:200].replace("\n", " ") + raise T2TunesError(f"Response was not JSON for {url}: {preview!r}") from exc + + +def _iter_search_items(response: Any) -> Iterable[T2TunesSearchItem]: + if not isinstance(response, dict): + raise T2TunesError(f"Unexpected search response type: {type(response).__name__}") + for result in response.get("results") or []: + if not isinstance(result, dict): + continue + for hit in result.get("hits") or []: + if not isinstance(hit, dict): + continue + document = hit.get("document") + if not isinstance(document, dict): + continue + asin = str(document.get("asin") or "") + if not asin: + continue + yield T2TunesSearchItem( + asin=asin, + title=str(document.get("title") or ""), + artist_name=str(document.get("artistName") or ""), + item_type=str(document.get("__type") or ""), + album_name=str(document.get("albumName") or ""), + album_asin=str(document.get("albumAsin") or ""), + duration_seconds=int(document.get("duration") or 0), + isrc=str(document.get("isrc") or ""), + ) + + +def _media_response_to_stream_info(item: Dict[str, Any]) -> T2TunesStreamInfo: + stream_info = item.get("streamInfo") if isinstance(item.get("streamInfo"), dict) else {} + tags = item.get("tags") if isinstance(item.get("tags"), dict) else {} + return T2TunesStreamInfo( + asin=str(item.get("asin") or ""), + streamable=bool(item.get("streamable") if item.get("streamable") is not None else item.get("stremeable")), + codec=str(stream_info.get("codec") or ""), + format=str(stream_info.get("format") or ""), + sample_rate=stream_info.get("sampleRate") if isinstance(stream_info.get("sampleRate"), int) else None, + stream_url=str(stream_info.get("streamUrl") or ""), + has_decryption_key=bool(item.get("decryptionKey")), + title=str(tags.get("title") or ""), + artist=str(tags.get("artist") or ""), + album=str(tags.get("album") or ""), + isrc=str(tags.get("isrc") or ""), + ) + + +def _print_json(data: Any) -> None: + print(json.dumps(data, indent=2, sort_keys=True, default=lambda o: getattr(o, "__dict__", str(o)))) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Probe a T2Tunes/TripleTriple API instance.") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL) + parser.add_argument("--country", default="US") + parser.add_argument("--codec", default="flac", choices=("flac", "opus", "eac3")) + parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT_SECONDS) + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("status") + + search_parser = sub.add_parser("search") + search_parser.add_argument("query") + search_parser.add_argument("--types", default="track,album") + + metadata_parser = sub.add_parser("metadata") + metadata_parser.add_argument("asin") + + media_parser = sub.add_parser("media") + media_parser.add_argument("asin") + media_parser.add_argument("--probe-stream", action="store_true") + + smoke_parser = sub.add_parser("smoke") + smoke_parser.add_argument("query") + smoke_parser.add_argument("--probe-stream", action="store_true") + + args = parser.parse_args() + client = T2TunesClient(args.base_url, country=args.country, codec=args.codec, timeout=args.timeout) + + if args.command == "status": + _print_json(client.status()) + return 0 + + if args.command == "search": + _print_json([item.__dict__ for item in client.search(args.query, types=args.types)]) + return 0 + + if args.command == "metadata": + _print_json(client.album_metadata(args.asin)) + return 0 + + if args.command == "media": + streams = client.media_from_asin(args.asin) + payload = [stream.__dict__ for stream in streams] + if args.probe_stream: + for index, stream in enumerate(streams): + payload[index]["stream_probe"] = client.probe_stream(stream.stream_url) if stream.stream_url else {"ok": False} + _print_json(payload) + return 0 + + if args.command == "smoke": + status = client.status() + search_items = client.search(args.query) + first = next((item for item in search_items if item.is_track or item.is_album), None) + media = client.media_from_asin(first.asin) if first else [] + payload = { + "status": status, + "amazon_music_up": str(status.get("amazonMusic", "")).lower() == "up", + "result_count": len(search_items), + "first_result": first.__dict__ if first else None, + "media": [stream.__dict__ for stream in media[:3]], + } + if args.probe_stream and media and media[0].stream_url: + payload["first_stream_probe"] = client.probe_stream(media[0].stream_url) + _print_json(payload) + return 0 + + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/web_server.py b/web_server.py index db0d01f9..f3cad76f 100644 --- a/web_server.py +++ b/web_server.py @@ -40,7 +40,7 @@ logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, system-info, update check, etc. # Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release. -_SOULSYNC_BASE_VERSION = "2.5.2" +_SOULSYNC_BASE_VERSION = "2.6.0" def _build_version_string(): """Append short commit hash to version when available (e.g. 2.35+abc1234).""" @@ -115,10 +115,7 @@ from core.imports.context import ( get_import_clean_album, get_import_clean_title, get_import_context_album, - get_import_context_artist, get_import_original_search, - get_import_track_info, - normalize_import_context, ) from core.wishlist.payloads import ( build_cancelled_task_wishlist_payload as _build_cancelled_task_wishlist_payload, @@ -161,23 +158,25 @@ from core.wishlist.state import ( is_wishlist_actually_processing as _is_wishlist_actually_processing, reset_flag_if_stuck as _reset_wishlist_flag_if_stuck, ) -from core.imports.album import ( - build_album_import_context, - build_album_import_match_payload, - resolve_album_artist_context, -) from core.imports.album_naming import resolve_album_group as _resolve_album_group +from core.imports.album import build_album_import_match_payload from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata from core.imports.staging import ( - get_import_suggestions_cache, - get_primary_source, get_staging_path, read_staging_file_metadata, - refresh_import_suggestions_cache, - search_import_albums, - search_import_tracks, start_import_suggestions_cache, ) +from core.imports.routes import ImportRouteRuntime as _ImportRouteRuntime +from core.imports.routes import album_match as _import_album_match +from core.imports.routes import album_process as _import_album_process +from core.imports.routes import process_single_import_file as _import_process_single_import_file +from core.imports.routes import search_albums as _import_search_albums +from core.imports.routes import search_tracks as _import_search_tracks +from core.imports.routes import singles_process as _import_singles_process +from core.imports.routes import staging_files as _import_staging_files +from core.imports.routes import staging_groups as _import_staging_groups +from core.imports.routes import staging_hints as _import_staging_hints +from core.imports.routes import staging_suggestions as _import_staging_suggestions from core.imports.paths import build_final_path_for_track as _build_final_path_for_track from core.imports.pipeline import build_import_pipeline_runtime as _build_import_pipeline_runtime from core.metadata.common import get_file_lock @@ -232,6 +231,7 @@ from core.genius_worker import GeniusWorker from core.tidal_worker import TidalWorker from core.qobuz_worker import QobuzWorker from core.hydrabase_worker import HydrabaseWorker +from core.amazon_worker import AmazonWorker from core.hydrabase_client import HydrabaseClient from core.automation_engine import AutomationEngine @@ -475,10 +475,16 @@ def _add_discover_cache_headers(response): def get_current_profile_id() -> int: - """Get the current profile ID from Flask g context or default to 1""" + """Get the current profile ID from Flask g context or default to 1. + + Background callers (automation engine, sync threads, watchlist + scanner) have no request context, so `g.profile_id` raises + `RuntimeError("Working outside of application context")` rather + than `AttributeError`. Catch both so non-request callers degrade + to the admin profile instead of crashing the handler.""" try: return g.profile_id - except AttributeError: + except (AttributeError, RuntimeError): return 1 @@ -768,6 +774,15 @@ db_update_state = { _db_update_automation_id = None # Set when automation triggers DB update, used by callbacks db_update_lock = threading.Lock() + +def _set_db_update_automation_id(value): + """Setter exposed to extracted automation handlers — keeps the + legacy `_db_update_automation_id` global in sync so the live + DB-update progress callbacks below (which still read the global + directly) emit against the right automation card.""" + global _db_update_automation_id + _db_update_automation_id = value + # Quality Scanner state quality_scanner_state = { "status": "idle", # idle, running, finished, error @@ -907,1501 +922,87 @@ _scan_library_automation_id = None def _register_automation_handlers(): - """Register real SoulSync action handlers with the automation engine.""" + """Register real SoulSync action handlers with the automation engine. + + Per-handler bodies live in ``core.automation.handlers``. This + function wires the dependency-injection surface (clients, + callables, mutable state) into a single ``AutomationDeps`` object + and hands it to ``register_all``. + + NOTE: extraction is in progress. The first batch of handlers + (``process_wishlist`` / ``scan_watchlist`` / ``scan_library``) + has been moved to ``core/automation/handlers/``; the remaining + closures still live below until subsequent commits in the same + branch finish the lift. + """ if not automation_engine: return - def _auto_process_wishlist(config): - try: - _process_wishlist_automatically(automation_id=config.get('_automation_id')) - return {'status': 'completed'} - except Exception as e: - return {'status': 'error', 'error': str(e)} - # Note: wishlist processing is async (batch submitted to executor), stats come via batch completion + from core.automation.deps import AutomationDeps, AutomationState + from core.automation.handlers import register_all as _register_extracted_handlers + + # Mutable shared state previously lived as module-level globals + # (`_scan_library_automation_id`, `_pipeline_running`, etc). + # Keeping the legacy globals AS WELL as the new state object during + # the transitional period so the not-yet-extracted closures still + # work; they'll be removed once the rest of the lift is done. + _automation_state = AutomationState() + + from core.watchlist_scanner import get_watchlist_scanner as _get_watchlist_scanner_fn + + _automation_deps = AutomationDeps( + engine=automation_engine, + state=_automation_state, + config_manager=config_manager, + update_progress=_update_automation_progress, + logger=logger, + get_database=get_database, + spotify_client=spotify_client, + tidal_client=tidal_client, + web_scan_manager=web_scan_manager, + process_wishlist_automatically=_process_wishlist_automatically, + process_watchlist_scan_automatically=_process_watchlist_scan_automatically, + is_wishlist_actually_processing=is_wishlist_actually_processing, + is_watchlist_actually_scanning=is_watchlist_actually_scanning, + get_watchlist_scan_state=lambda: watchlist_scan_state, + run_playlist_discovery_worker=_run_playlist_discovery_worker, + run_sync_task=_run_sync_task, + load_sync_status_file=_load_sync_status_file, + get_deezer_client=_get_deezer_client, + parse_youtube_playlist=parse_youtube_playlist, + get_sync_states=lambda: sync_states, + set_db_update_automation_id=_set_db_update_automation_id, + get_db_update_state=lambda: db_update_state, + db_update_lock=db_update_lock, + db_update_executor=db_update_executor, + run_db_update_task=_run_db_update_task, + run_deep_scan_task=_run_deep_scan_task, + get_duplicate_cleaner_state=lambda: duplicate_cleaner_state, + duplicate_cleaner_lock=duplicate_cleaner_lock, + duplicate_cleaner_executor=duplicate_cleaner_executor, + run_duplicate_cleaner=_run_duplicate_cleaner, + get_quality_scanner_state=lambda: quality_scanner_state, + quality_scanner_lock=quality_scanner_lock, + quality_scanner_executor=quality_scanner_executor, + run_quality_scanner=_run_quality_scanner, + download_orchestrator=download_orchestrator, + run_async=run_async, + tasks_lock=tasks_lock, + get_download_batches=lambda: download_batches, + get_download_tasks=lambda: download_tasks, + sweep_empty_download_directories=_sweep_empty_download_directories, + get_staging_path=get_staging_path, + docker_resolve_path=docker_resolve_path, + get_current_profile_id=get_current_profile_id, + get_watchlist_scanner=_get_watchlist_scanner_fn, + get_app=lambda: app, + get_beatport_data_cache=lambda: beatport_data_cache, + init_automation_progress=_init_automation_progress, + record_progress_history=_auto_progress.record_history, + build_personalized_manager=_build_personalized_manager, + ) + _register_extracted_handlers(_automation_deps) - def _auto_scan_watchlist(config): - try: - pre_state_id = id(watchlist_scan_state) - _process_watchlist_scan_automatically( - automation_id=config.get('_automation_id'), - profile_id=config.get('_profile_id') - ) - # Only report stats if a fresh scan actually ran (state dict was reassigned) - if id(watchlist_scan_state) != pre_state_id: - summary = watchlist_scan_state.get('summary', {}) - return { - 'status': 'completed', - 'artists_scanned': summary.get('total_artists', 0), - 'successful_scans': summary.get('successful_scans', 0), - 'new_tracks_found': summary.get('new_tracks_found', 0), - 'tracks_added_to_wishlist': summary.get('tracks_added_to_wishlist', 0), - } - return {'status': 'completed'} - except Exception as e: - return {'status': 'error', 'error': str(e)} - - def _auto_scan_library(config): - global _scan_library_automation_id - automation_id = config.get('_automation_id') - - if not web_scan_manager: - return {'status': 'error', 'reason': 'Scan manager not available'} - - # If another automation is already tracking the scan, just forward the request - if _scan_library_automation_id is not None: - web_scan_manager.request_scan('Automation trigger (additional batch)') - return {'status': 'skipped', 'reason': 'Scan already being tracked'} - - _scan_library_automation_id = automation_id - - try: - result = web_scan_manager.request_scan('Automation trigger') - scan_status_val = result.get('status', 'unknown') - - if scan_status_val == 'queued': - _update_automation_progress(automation_id, - log_line='Scan already in progress — waiting for completion', log_type='info') - else: - delay = result.get('delay_seconds', 60) - _update_automation_progress(automation_id, - log_line=f'Scan scheduled (debounce: {delay}s)', log_type='info') - - # Unified polling loop — handles debounce → scanning → idle transitions - poll_start = time.time() - scan_started = (scan_status_val == 'queued') # Already scanning if queued - while time.time() - poll_start < 1800: # Max 30 min overall - status = web_scan_manager.get_scan_status() - st = status.get('status') - - if st == 'idle': - break # Scan completed (or finished before we started polling) - - elif st == 'scheduled': - elapsed = int(time.time() - poll_start) - _update_automation_progress(automation_id, - phase=f'Waiting for scan to start... ({elapsed}s)', - progress=min(int(elapsed / 60 * 10), 14)) - time.sleep(2) - - elif st == 'scanning': - if not scan_started: - scan_started = True - _update_automation_progress(automation_id, progress=15, - log_line='Scan triggered on media server', log_type='success') - elapsed = status.get('elapsed_seconds', 0) - max_time = status.get('max_time_seconds', 300) - pct = min(15 + int(elapsed / max_time * 80), 95) - mins, secs = divmod(elapsed, 60) - _update_automation_progress(automation_id, - phase=f'Library scan in progress... ({mins}m {secs}s)', - progress=pct) - time.sleep(5) - - else: - time.sleep(2) # Unknown status, avoid tight loop - else: - # 30-min timeout reached - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Library scan timed out after 30 minutes', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - elapsed = round(time.time() - poll_start, 1) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line='Library scan completed', log_type='success') - - return {'status': 'completed', '_manages_own_progress': True, 'scan_duration_seconds': elapsed} - - except Exception as e: - _update_automation_progress(automation_id, status='error', - phase='Error', log_line=str(e), log_type='error') - return {'status': 'error', 'error': str(e), '_manages_own_progress': True} - - finally: - _scan_library_automation_id = None - - # Self-guards only — prevent duplicate runs of the same operation, - # but allow wishlist processing and watchlist scanning to run concurrently. - # Downloads use bandwidth (Soulseek/Tidal/etc), scans use API calls — different resources. - # The per-call rate limiter handles any API contention during post-processing. - automation_engine.register_action_handler('process_wishlist', _auto_process_wishlist, - guard_fn=lambda: is_wishlist_actually_processing()) - automation_engine.register_action_handler('scan_watchlist', _auto_scan_watchlist, - guard_fn=lambda: is_watchlist_actually_scanning()) - automation_engine.register_action_handler('scan_library', _auto_scan_library, - lambda: _scan_library_automation_id is not None) - - def _auto_refresh_mirrored(config): - """Refresh mirrored playlist(s) from source.""" - db = get_database() - playlist_id = config.get('playlist_id') - refresh_all = config.get('all', False) - auto_id = config.get('_automation_id') - - if refresh_all: - playlists = db.get_mirrored_playlists() - elif playlist_id: - p = db.get_mirrored_playlist(int(playlist_id)) - playlists = [p] if p else [] - else: - return {'status': 'error', 'reason': 'No playlist specified'} - - # Filter out sources that can't be refreshed (no external API) - playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] - - refreshed = 0 - errors = [] - for idx, pl in enumerate(playlists): - try: - source = pl.get('source', '') - source_id = pl.get('source_playlist_id', '') - _update_automation_progress(auto_id, - progress=(idx / max(1, len(playlists))) * 100, - phase=f'Refreshing: "{pl.get("name", "")}"', - current_item=pl.get('name', '')) - tracks = None - - if source == 'spotify': - # Try authenticated API first, fall back to public embed scraper - if spotify_client and spotify_client.is_spotify_authenticated(): - playlist_obj = spotify_client.get_playlist_by_id(source_id) - if playlist_obj and playlist_obj.tracks: - tracks = [] - for t in playlist_obj.tracks: - artist_name = t.artists[0] if t.artists else '' - track_dict = { - 'track_name': t.name or '', - 'artist_name': str(artist_name), - 'album_name': t.album or '', - 'duration_ms': t.duration_ms or 0, - 'source_track_id': t.id or '', - } - # Spotify data IS official — auto-mark as discovered - if t.id: - _album_obj = {'name': t.album or ''} - if getattr(t, 'image_url', None): - _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] - track_dict['extra_data'] = json.dumps({ - 'discovered': True, - 'provider': 'spotify', - 'confidence': 1.0, - 'matched_data': { - 'id': t.id, - 'name': t.name or '', - 'artists': [{'name': str(a)} for a in (t.artists or [])], - 'album': _album_obj, - 'duration_ms': t.duration_ms or 0, - 'image_url': getattr(t, 'image_url', None), - } - }) - tracks.append(track_dict) - - # Fallback: public embed scraper (no auth needed) - if tracks is None: - try: - from core.spotify_public_scraper import scrape_spotify_embed - embed_data = scrape_spotify_embed('playlist', source_id) - if embed_data and not embed_data.get('error') and embed_data.get('tracks'): - embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' - tracks = [] - for t in embed_data['tracks']: - artist_names = [a['name'] for a in t.get('artists', [])] - artist_name = artist_names[0] if artist_names else '' - track_dict = { - 'track_name': t.get('name', ''), - 'artist_name': artist_name, - 'album_name': embed_album, - 'duration_ms': t.get('duration_ms', 0), - 'source_track_id': t.get('id', ''), - } - # Store Spotify track ID hint but don't mark discovered — - # Discover step needs to run for proper album art - if t.get('id'): - track_dict['extra_data'] = json.dumps({ - 'discovered': False, - 'spotify_hint': { - 'id': t['id'], - 'name': t.get('name', ''), - 'artists': t.get('artists', []), - } - }) - tracks.append(track_dict) - except Exception as e: - logger.warning(f"Spotify public scraper fallback failed for {source_id}: {e}") - - elif source == 'spotify_public': - # source_playlist_id is an MD5 hash; extract actual Spotify ID from stored description (URL) - try: - from core.spotify_public_scraper import parse_spotify_url, scrape_spotify_embed - spotify_url = pl.get('description', '') - parsed = parse_spotify_url(spotify_url) if spotify_url else None - - # If Spotify is authenticated, use the full API (auto-discovers with album art) - if parsed and parsed.get('type') == 'playlist' and spotify_client and spotify_client.is_spotify_authenticated(): - playlist_obj = spotify_client.get_playlist_by_id(parsed['id']) - if playlist_obj and playlist_obj.tracks: - tracks = [] - for t in playlist_obj.tracks: - artist_name = t.artists[0] if t.artists else '' - track_dict = { - 'track_name': t.name or '', - 'artist_name': str(artist_name), - 'album_name': t.album or '', - 'duration_ms': t.duration_ms or 0, - 'source_track_id': t.id or '', - } - if t.id: - _album_obj = {'name': t.album or ''} - if getattr(t, 'image_url', None): - _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] - track_dict['extra_data'] = json.dumps({ - 'discovered': True, - 'provider': 'spotify', - 'confidence': 1.0, - 'matched_data': { - 'id': t.id, - 'name': t.name or '', - 'artists': [{'name': str(a)} for a in (t.artists or [])], - 'album': _album_obj, - 'duration_ms': t.duration_ms or 0, - 'image_url': getattr(t, 'image_url', None), - } - }) - tracks.append(track_dict) - - # Fallback: public embed scraper (no auth or album-type URL) - if tracks is None and parsed: - embed_data = scrape_spotify_embed(parsed['type'], parsed['id']) - if embed_data and not embed_data.get('error') and embed_data.get('tracks'): - embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' - tracks = [] - for t in embed_data['tracks']: - artist_names = [a['name'] for a in t.get('artists', [])] - artist_name = artist_names[0] if artist_names else '' - tracks.append({ - 'track_name': t.get('name', ''), - 'artist_name': artist_name, - 'album_name': embed_album, - 'duration_ms': t.get('duration_ms', 0), - 'source_track_id': t.get('id', ''), - }) - # No extra_data — let preservation code keep existing discovery data - except Exception as e: - logger.warning(f"Spotify public playlist refresh failed for {source_id}: {e}") - - elif source == 'deezer': - try: - deezer = _get_deezer_client() - playlist_data = deezer.get_playlist(source_id) - if playlist_data and playlist_data.get('tracks'): - tracks = [] - for t in playlist_data['tracks']: - artist_name = t['artists'][0] if t.get('artists') else '' - tracks.append({ - 'track_name': t.get('name', ''), - 'artist_name': str(artist_name), - 'album_name': t.get('album', ''), - 'duration_ms': t.get('duration_ms', 0), - 'source_track_id': str(t.get('id', '')), - }) - except Exception as e: - logger.warning(f"Deezer playlist refresh failed for {source_id}: {e}") - - elif source == 'tidal': - if not tidal_client or not tidal_client.is_authenticated(): - logger.warning(f"Tidal not authenticated — skipping refresh for '{pl.get('name', '')}'") - _update_automation_progress(auto_id, - log_line=f'Skipped "{pl.get("name", "")}" — Tidal not authenticated', log_type='skip') - continue - full_playlist = tidal_client.get_playlist(source_id) - if full_playlist and full_playlist.tracks: - tracks = [] - for t in full_playlist.tracks: - artist_name = t.artists[0] if t.artists else '' - tracks.append({ - 'track_name': t.name or '', - 'artist_name': str(artist_name), - 'album_name': t.album or '', - 'duration_ms': t.duration_ms or 0, - 'source_track_id': t.id or '', - }) - - elif source == 'youtube': - # source_playlist_id is now a deterministic hash; use stored description (original URL) for refresh - yt_url = pl.get('description', '') or f"https://www.youtube.com/playlist?list={source_id}" - playlist_data = parse_youtube_playlist(yt_url) - if playlist_data and playlist_data.get('tracks'): - tracks = [] - for t in playlist_data['tracks']: - artist_name = t['artists'][0] if t.get('artists') else '' - tracks.append({ - 'track_name': t.get('name', ''), - 'artist_name': str(artist_name), - 'album_name': '', - 'duration_ms': t.get('duration_ms', 0), - 'source_track_id': t.get('id', ''), - }) - - if tracks is not None: - # Compare old vs new track IDs to detect changes - old_tracks = db.get_mirrored_playlist_tracks(pl['id']) if pl.get('id') else [] - old_ids = {t.get('source_track_id') for t in old_tracks if t.get('source_track_id')} - new_ids = {t.get('source_track_id') for t in tracks if t.get('source_track_id')} - - # Preserve existing discovery extra_data for tracks that still exist - old_extra_map = db.get_mirrored_tracks_extra_data_map(pl['id']) if pl.get('id') else {} - for t in tracks: - sid = t.get('source_track_id', '') - if sid and sid in old_extra_map and 'extra_data' not in t: - t['extra_data'] = old_extra_map[sid] - - db.mirror_playlist( - source=source, - source_playlist_id=source_id, - name=pl['name'], - tracks=tracks, - profile_id=pl.get('profile_id', 1), - owner=pl.get('owner'), - image_url=pl.get('image_url'), - ) - refreshed += 1 - - # Emit playlist_changed if tracks actually changed - if old_ids != new_ids: - added_count = len(new_ids - old_ids) - removed_count = len(old_ids - new_ids) - logger.info(f"[AUTOMATION] Playlist changed: '{pl.get('name', '')}' — {added_count} added, {removed_count} removed (old={len(old_ids)}, new={len(new_ids)})") - _update_automation_progress(auto_id, - log_line=f'"{pl.get("name", "")}" — {added_count} added, {removed_count} removed', log_type='success') - try: - if automation_engine: - automation_engine.emit('playlist_changed', { - 'playlist_name': pl.get('name', ''), - 'playlist_id': str(pl.get('id', '')), - 'old_count': str(len(old_ids)), - 'new_count': str(len(new_ids)), - 'added': str(added_count), - 'removed': str(removed_count), - }) - except Exception as e: - logger.debug("playlist_synced automation emit failed: %s", e) - else: - logger.warning(f"[AUTOMATION] No changes: '{pl.get('name', '')}' (tracks={len(old_ids)})") - _update_automation_progress(auto_id, - log_line=f'No changes: "{pl.get("name", "")}"', log_type='skip') - except Exception as e: - errors.append(f"{pl.get('name', '?')}: {str(e)}") - _update_automation_progress(auto_id, - log_line=f'Error: {pl.get("name", "?")} — {str(e)}', log_type='error') - return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors))} - - def _auto_sync_playlist(config): - """Sync a mirrored playlist to media server. - Uses discovered metadata when available, skips undiscovered tracks. - When triggered on a schedule, skips if nothing changed since last sync.""" - auto_id = config.get('_automation_id') - playlist_id = config.get('playlist_id') - if not playlist_id: - return {'status': 'error', 'reason': 'No playlist specified'} - - db = get_database() - pl = db.get_mirrored_playlist(int(playlist_id)) - if not pl: - return {'status': 'error', 'reason': 'Playlist not found'} - - tracks = db.get_mirrored_playlist_tracks(int(playlist_id)) - if not tracks: - return {'status': 'error', 'reason': 'No tracks in playlist'} - - # Count currently discovered tracks for smart-skip check - current_discovered = 0 - for t in tracks: - extra = {} - if t.get('extra_data'): - try: - extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] - except (json.JSONDecodeError, TypeError): - pass - if extra.get('discovered') and extra.get('matched_data'): - current_discovered += 1 - - # Convert mirrored tracks to format expected by _run_sync_task - # Use discovered metadata when available, skip undiscovered tracks - tracks_json = [] - skipped_count = 0 - - for t in tracks: - # Parse extra_data for discovery info - extra = {} - if t.get('extra_data'): - try: - extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] - except (json.JSONDecodeError, TypeError): - pass - - if extra.get('discovered') and extra.get('matched_data'): - # Use official discovered metadata - md = extra['matched_data'] - album_raw = md.get('album', '') - album_obj = album_raw if isinstance(album_raw, dict) else {'name': album_raw or ''} - _track_entry = { - 'name': md.get('name', ''), - 'artists': md.get('artists', [{'name': t.get('artist_name', '')}]), - 'album': album_obj, - 'duration_ms': md.get('duration_ms', 0), - 'id': md.get('id', ''), - } - if md.get('track_number'): - _track_entry['track_number'] = md['track_number'] - if md.get('disc_number'): - _track_entry['disc_number'] = md['disc_number'] - tracks_json.append(_track_entry) - else: - # NOT discovered — try to include using available metadata so the - # track can still be searched on Soulseek and added to wishlist. - # Without this, failed discovery blocks the entire download pipeline. - # - # Priority: spotify_hint (has real Spotify ID from embed scraper) - # > raw playlist fields (only if source_track_id is valid) - hint = extra.get('spotify_hint', {}) - # Build album object with cover art from the mirrored playlist track - track_image = (t.get('image_url') or '').strip() - album_obj = { - 'name': (t.get('album_name') or '').strip(), - 'images': [{'url': track_image, 'height': 300, 'width': 300}] if track_image else [], - } - - if hint.get('id') and hint.get('name'): - # spotify_hint has proper Spotify track ID + metadata from embed scraper - hint_artists = hint.get('artists', []) - if hint_artists and isinstance(hint_artists[0], str): - hint_artists = [{'name': a} for a in hint_artists] - elif hint_artists and isinstance(hint_artists[0], dict): - pass # Already in correct format - else: - hint_artists = [{'name': t.get('artist_name', '')}] - tracks_json.append({ - 'name': hint['name'], - 'artists': hint_artists, - 'album': album_obj, - 'duration_ms': t.get('duration_ms', 0), - 'id': hint['id'], - }) - elif t.get('source_track_id') and (t.get('track_name') or '').strip(): - # Has a valid source ID and track name — usable for wishlist - tracks_json.append({ - 'name': t['track_name'].strip(), - 'artists': [{'name': (t.get('artist_name') or '').strip() or 'Unknown Artist'}], - 'album': album_obj, - 'duration_ms': t.get('duration_ms', 0), - 'id': t['source_track_id'], - }) - else: - skipped_count += 1 # No usable ID or name — truly can't process - - if not tracks_json: - _update_automation_progress(auto_id, - log_line=f'No discovered tracks — {skipped_count} need discovery first', log_type='skip') - return { - 'status': 'skipped', - 'reason': f'No discovered tracks to sync ({skipped_count} tracks need discovery first)', - 'skipped_tracks': str(skipped_count), - } - - # Preflight: hash the track list and compare against last sync - # Skip if the exact same set of tracks was already synced and all matched - import hashlib - track_ids_str = ','.join(sorted(t.get('id', '') for t in tracks_json)) - tracks_hash = hashlib.md5(track_ids_str.encode()).hexdigest() - - sync_id_key = f"auto_mirror_{playlist_id}" - try: - sync_statuses = _load_sync_status_file() - last_status = sync_statuses.get(sync_id_key, {}) - last_hash = last_status.get('tracks_hash', '') - last_matched = last_status.get('matched_tracks', -1) - - if (last_hash == tracks_hash and - last_matched >= len(tracks_json)): - # Exact same tracks, all matched last time — nothing to do - _update_automation_progress(auto_id, - log_line=f'All {len(tracks_json)} tracks unchanged since last sync — skipping', - log_type='skip') - return { - 'status': 'skipped', - 'reason': f'All {len(tracks_json)} tracks unchanged since last sync', - } - except Exception as e: - logger.debug("mirror sync last-status read: %s", e) - - _update_automation_progress(auto_id, progress=50, - phase=f'Syncing "{pl["name"]}"', - log_line=f'{len(tracks_json)} discovered, {skipped_count} skipped', - log_type='info') - - sync_id = f"auto_mirror_{playlist_id}" - _update_automation_progress(auto_id, progress=90, - log_line=f'Starting sync: {len(tracks_json)} tracks', log_type='success') - threading.Thread( - target=_run_sync_task, - args=(sync_id, pl['name'], tracks_json, auto_id, 1, pl.get('image_url', '')), - daemon=True, - name=f'auto-sync-{playlist_id}' - ).start() - return { - 'status': 'started', - 'playlist_name': pl['name'], - 'discovered_tracks': str(len(tracks_json)), - 'skipped_tracks': str(skipped_count), - '_manages_own_progress': True, - } - - def _auto_discover_playlist(config): - """Discover official Spotify/iTunes metadata for mirrored playlist tracks.""" - db = get_database() - playlist_id = config.get('playlist_id') - discover_all = config.get('all', False) - - if discover_all: - playlists = db.get_mirrored_playlists() - elif playlist_id: - p = db.get_mirrored_playlist(int(playlist_id)) - playlists = [p] if p else [] - else: - return {'status': 'error', 'reason': 'No playlist specified'} - - if not playlists: - return {'status': 'error', 'reason': 'No playlists found'} - - threading.Thread( - target=_run_playlist_discovery_worker, - args=(playlists, config.get('_automation_id')), - daemon=True, - name='auto-discover-playlist' - ).start() - names = ', '.join(p['name'] for p in playlists[:3]) - return {'status': 'started', 'playlist_count': str(len(playlists)), 'playlists': names, - '_manages_own_progress': True} - - # --- Playlist Pipeline: single automation for full lifecycle --- - _pipeline_running = False - - def _pipeline_guard(): - return _pipeline_running - - def _auto_playlist_pipeline(config): - """Full playlist lifecycle: refresh → discover → sync → wishlist. - Runs all 4 phases sequentially in one automation, reporting progress throughout.""" - nonlocal _pipeline_running - _pipeline_running = True - automation_id = config.get('_automation_id') - pipeline_start = time.time() - - try: - db = get_database() - playlist_id = config.get('playlist_id') - process_all = config.get('all', False) - skip_wishlist = config.get('skip_wishlist', False) - - # Resolve playlists - if process_all: - playlists = db.get_mirrored_playlists() - elif playlist_id: - p = db.get_mirrored_playlist(int(playlist_id)) - playlists = [p] if p else [] - else: - _pipeline_running = False - return {'status': 'error', 'error': 'No playlist specified'} - - playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] - if not playlists: - _pipeline_running = False - return {'status': 'error', 'error': 'No refreshable playlists found'} - - pl_names = ', '.join(p.get('name', '?') for p in playlists[:3]) - if len(playlists) > 3: - pl_names += f' (+{len(playlists) - 3} more)' - - _update_automation_progress(automation_id, progress=2, - phase=f'Pipeline: {len(playlists)} playlist(s)', - log_line=f'Starting pipeline for: {pl_names}', log_type='info') - - # ── PHASE 1: REFRESH ────────────────────────────────────────── - _update_automation_progress(automation_id, progress=3, - phase='Phase 1/4: Refreshing playlists...', - log_line='Phase 1: Refresh', log_type='info') - - refresh_config = dict(config) - refresh_config['_automation_id'] = None # Don't let sub-handler hijack pipeline progress - refresh_result = _auto_refresh_mirrored(refresh_config) - refreshed = int(refresh_result.get('refreshed', 0)) - refresh_errors = int(refresh_result.get('errors', 0)) - - _update_automation_progress(automation_id, progress=25, - phase='Phase 1/4: Refresh complete', - log_line=f'Phase 1 done: {refreshed} refreshed, {refresh_errors} errors', - log_type='success' if refresh_errors == 0 else 'warning') - - # ── PHASE 2: DISCOVER ───────────────────────────────────────── - _update_automation_progress(automation_id, progress=26, - phase='Phase 2/4: Discovering metadata...', - log_line='Phase 2: Discover', log_type='info') - - # Reload playlists (refresh may have updated them) - if process_all: - disc_playlists = db.get_mirrored_playlists() - else: - disc_playlists = [db.get_mirrored_playlist(int(playlist_id))] - disc_playlists = [p for p in disc_playlists if p] - - # Run discovery in a thread and wait for it - disc_done = threading.Event() - disc_result = {'discovered': 0, 'failed': 0, 'skipped': 0, 'total': 0} - - def _disc_wrapper(pls): - try: - # The worker updates automation_progress internally, - # but we pass None so it doesn't conflict with our pipeline progress - _run_playlist_discovery_worker(pls, automation_id=None) - except Exception as e: - logger.error(f"[Pipeline] Discovery error: {e}") - finally: - disc_done.set() - - threading.Thread(target=_disc_wrapper, args=(disc_playlists,), daemon=True, - name='pipeline-discover').start() - - # Poll for completion with progress updates - poll_start = time.time() - while not disc_done.wait(timeout=3): - elapsed = int(time.time() - poll_start) - _update_automation_progress(automation_id, progress=min(26 + elapsed // 4, 54), - phase=f'Phase 2/4: Discovering... ({elapsed}s)') - if elapsed > 3600: # 1hr safety timeout - _update_automation_progress(automation_id, - log_line='Discovery timed out after 1 hour', log_type='warning') - break - - _update_automation_progress(automation_id, progress=55, - phase='Phase 2/4: Discovery complete', - log_line='Phase 2 done: discovery complete', log_type='success') - - # ── PHASE 3: SYNC ───────────────────────────────────────────── - _update_automation_progress(automation_id, progress=56, - phase='Phase 3/4: Syncing to server...', - log_line='Phase 3: Sync', log_type='info') - - total_synced = 0 - total_skipped = 0 - sync_errors = 0 - - for pl_idx, pl in enumerate(playlists): - pl_id = pl.get('id') - if not pl_id: - continue - - # Build sync config for this playlist (reuse existing sync handler) - sync_config = { - 'playlist_id': str(pl_id), - '_automation_id': None, # Don't let sync handler hijack our progress - } - sync_result = _auto_sync_playlist(sync_config) - sync_status = sync_result.get('status', '') - - if sync_status == 'started': - # Sync launched a background thread — wait for it - sync_id = f"auto_mirror_{pl_id}" - sync_poll_start = time.time() - while time.time() - sync_poll_start < 600: # 10 min per playlist max - if sync_id in sync_states and sync_states[sync_id].get('status') in ('finished', 'complete', 'error', 'failed'): - break - time.sleep(2) - elapsed = int(time.time() - sync_poll_start) - sub_progress = 56 + ((pl_idx + 1) / max(1, len(playlists))) * 29 - _update_automation_progress(automation_id, progress=min(int(sub_progress), 84), - phase=f'Phase 3/4: Syncing "{pl.get("name", "")}" ({elapsed}s)') - - # Check result - ss = sync_states.get(sync_id, {}) - ss_result = ss.get('result', ss.get('progress', {})) - matched = ss_result.get('matched_tracks', 0) if isinstance(ss_result, dict) else 0 - total_synced += int(matched) if matched else 0 - _update_automation_progress(automation_id, - log_line=f'Synced "{pl.get("name", "")}": {matched} tracks matched', - log_type='success') - - elif sync_status == 'skipped': - total_skipped += 1 - reason = sync_result.get('reason', 'unchanged') - _update_automation_progress(automation_id, - log_line=f'Skipped "{pl.get("name", "")}": {reason}', - log_type='skip') - elif sync_status == 'error': - sync_errors += 1 - _update_automation_progress(automation_id, - log_line=f'Sync error "{pl.get("name", "")}": {sync_result.get("reason", "unknown")}', - log_type='error') - - _update_automation_progress(automation_id, progress=85, - phase='Phase 3/4: Sync complete', - log_line=f'Phase 3 done: {total_synced} matched, {total_skipped} skipped, {sync_errors} errors', - log_type='success' if sync_errors == 0 else 'warning') - - # ── PHASE 4: WISHLIST ───────────────────────────────────────── - wishlist_queued = 0 - if not skip_wishlist: - _update_automation_progress(automation_id, progress=86, - phase='Phase 4/4: Processing wishlist...', - log_line='Phase 4: Wishlist', log_type='info') - - try: - if not is_wishlist_actually_processing(): - _process_wishlist_automatically(automation_id=None) - _update_automation_progress(automation_id, - log_line='Wishlist processing triggered', log_type='success') - wishlist_queued = 1 - else: - _update_automation_progress(automation_id, - log_line='Wishlist already running — skipped', log_type='skip') - except Exception as e: - _update_automation_progress(automation_id, - log_line=f'Wishlist error: {e}', log_type='warning') - else: - _update_automation_progress(automation_id, progress=86, - log_line='Phase 4: Wishlist skipped (disabled)', log_type='skip') - - # ── COMPLETE ────────────────────────────────────────────────── - duration = int(time.time() - pipeline_start) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Pipeline complete', - log_line=f'Pipeline finished in {duration // 60}m {duration % 60}s', - log_type='success') - - _pipeline_running = False - return { - 'status': 'completed', - '_manages_own_progress': True, - 'playlists_refreshed': str(refreshed), - 'tracks_discovered': 'completed', - 'tracks_synced': str(total_synced), - 'sync_skipped': str(total_skipped), - 'wishlist_queued': str(wishlist_queued), - 'duration_seconds': str(duration), - } - - except Exception as e: - _pipeline_running = False - _update_automation_progress(automation_id, status='error', progress=100, - phase='Pipeline error', - log_line=f'Pipeline failed: {e}', log_type='error') - return {'status': 'error', 'error': str(e), '_manages_own_progress': True} - - automation_engine.register_action_handler('refresh_mirrored', _auto_refresh_mirrored) - automation_engine.register_action_handler('sync_playlist', _auto_sync_playlist) - automation_engine.register_action_handler('discover_playlist', _auto_discover_playlist) - automation_engine.register_action_handler('playlist_pipeline', _auto_playlist_pipeline, _pipeline_guard) - - # --- Phase 3 action handlers --- - - def _auto_start_database_update(config): - global _db_update_automation_id - automation_id = config.get('_automation_id') - if db_update_state.get('status') == 'running': - return {'status': 'skipped', 'reason': 'Database update already running'} - _db_update_automation_id = automation_id - full = config.get('full_refresh', False) - active_server = config_manager.get_active_media_server() - with db_update_lock: - db_update_state.update({ - "status": "running", "phase": "Initializing...", - "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "" - }) - db_update_executor.submit(_run_db_update_task, full, active_server) - - # Monitor DB update progress (callbacks handle card updates, we just block until done) - time.sleep(1) - poll_start = time.time() - last_progress_time = time.time() - last_progress_val = 0 - while time.time() - poll_start < 7200: # Max 2 hours - time.sleep(3) - with db_update_lock: - status = db_update_state.get('status', 'idle') - current_progress = db_update_state.get('progress', 0) - if status != 'running': - break - # Track stall detection — if no progress change in 10 minutes, warn - if current_progress != last_progress_val: - last_progress_val = current_progress - last_progress_time = time.time() - elif time.time() - last_progress_time > 600: - _update_automation_progress(automation_id, - log_line='Database update appears stalled — waiting...', log_type='warning') - last_progress_time = time.time() # Reset so warning repeats every 10 min - else: - # 2-hour timeout reached - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Database update timed out after 2 hours', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - # Finished/error callback already updated the card — return matching status - with db_update_lock: - final_status = db_update_state.get('status', 'unknown') - if final_status == 'error': - return {'status': 'error', 'reason': db_update_state.get('error_message', 'Unknown error'), '_manages_own_progress': True} - with db_update_lock: - stats = { - 'status': 'completed', 'full_refresh': str(full), '_manages_own_progress': True, - 'artists': db_update_state.get('total', 0), - 'albums': db_update_state.get('total_albums', 0), - 'tracks': db_update_state.get('total_tracks', 0), - 'removed_artists': db_update_state.get('removed_artists', 0), - 'removed_albums': db_update_state.get('removed_albums', 0), - 'removed_tracks': db_update_state.get('removed_tracks', 0), - } - return stats - - def _auto_deep_scan_library(config): - global _db_update_automation_id - automation_id = config.get('_automation_id') - if db_update_state.get('status') == 'running': - return {'status': 'skipped', 'reason': 'Database update already running'} - _db_update_automation_id = automation_id - active_server = config_manager.get_active_media_server() - with db_update_lock: - db_update_state.update({ - "status": "running", "phase": "Deep scan: Initializing...", - "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "" - }) - db_update_executor.submit(_run_deep_scan_task, active_server) - - # Monitor progress (callbacks handle card updates, we just block until done) - time.sleep(1) - poll_start = time.time() - last_progress_time = time.time() - last_progress_val = 0 - while time.time() - poll_start < 7200: # Max 2 hours - time.sleep(3) - with db_update_lock: - status = db_update_state.get('status', 'idle') - current_progress = db_update_state.get('progress', 0) - if status != 'running': - break - if current_progress != last_progress_val: - last_progress_val = current_progress - last_progress_time = time.time() - elif time.time() - last_progress_time > 600: - _update_automation_progress(automation_id, - log_line='Deep scan appears stalled — waiting...', log_type='warning') - last_progress_time = time.time() - else: - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Deep scan timed out after 2 hours', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - with db_update_lock: - final_status = db_update_state.get('status', 'unknown') - if final_status == 'error': - return {'status': 'error', 'reason': db_update_state.get('error_message', 'Unknown error'), '_manages_own_progress': True} - with db_update_lock: - stats = { - 'status': 'completed', '_manages_own_progress': True, - 'artists': db_update_state.get('total', 0), - 'albums': db_update_state.get('total_albums', 0), - 'tracks': db_update_state.get('total_tracks', 0), - 'removed_artists': db_update_state.get('removed_artists', 0), - 'removed_albums': db_update_state.get('removed_albums', 0), - 'removed_tracks': db_update_state.get('removed_tracks', 0), - } - return stats - - def _auto_run_duplicate_cleaner(config): - automation_id = config.get('_automation_id') - if duplicate_cleaner_state.get('status') == 'running': - return {'status': 'skipped', 'reason': 'Duplicate cleaner already running'} - - # Pre-set status before submit so polling loop doesn't see stale 'finished' from last run - with duplicate_cleaner_lock: - duplicate_cleaner_state["status"] = "running" - duplicate_cleaner_executor.submit(_run_duplicate_cleaner) - _update_automation_progress(automation_id, - log_line='Duplicate cleaner started', log_type='info') - - # Monitor duplicate cleaner progress (max 2 hours) - time.sleep(1) # Brief pause for executor to start - poll_start = time.time() - while time.time() - poll_start < 7200: - time.sleep(3) - status = duplicate_cleaner_state.get('status', 'idle') - if status not in ('running',): - break - phase = duplicate_cleaner_state.get('phase', 'Scanning...') - progress = duplicate_cleaner_state.get('progress', 0) - scanned = duplicate_cleaner_state.get('files_scanned', 0) - total = duplicate_cleaner_state.get('total_files', 0) - _update_automation_progress(automation_id, - phase=phase, progress=progress, - processed=scanned, total=total) - else: - # 2-hour timeout reached - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Duplicate cleaner timed out after 2 hours', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - # Check actual exit status (could be 'finished' or 'error') - final_status = duplicate_cleaner_state.get('status', 'idle') - if final_status == 'error': - err = duplicate_cleaner_state.get('error_message', 'Unknown error') - _update_automation_progress(automation_id, status='error', progress=100, - phase='Error', log_line=err, log_type='error') - return {'status': 'error', 'reason': err, '_manages_own_progress': True} - - dupes = duplicate_cleaner_state.get('duplicates_found', 0) - removed = duplicate_cleaner_state.get('deleted', 0) - space_freed = duplicate_cleaner_state.get('space_freed', 0) - scanned = duplicate_cleaner_state.get('files_scanned', 0) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line=f'Found {dupes} duplicates, removed {removed} files', log_type='success') - return { - 'status': 'completed', '_manages_own_progress': True, - 'files_scanned': scanned, - 'duplicates_found': dupes, - 'files_deleted': removed, - 'space_freed_mb': round(space_freed / (1024 * 1024), 1), - } - - def _auto_clear_quarantine(config): - import shutil as _shutil - automation_id = config.get('_automation_id') - quarantine_path = os.path.join(docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), 'ss_quarantine') - if not os.path.exists(quarantine_path): - _update_automation_progress(automation_id, - log_line='No quarantine folder found', log_type='info') - return {'status': 'completed', 'removed': '0'} - removed = 0 - for f in os.listdir(quarantine_path): - fp = os.path.join(quarantine_path, f) - try: - if os.path.isfile(fp): - os.remove(fp) - removed += 1 - elif os.path.isdir(fp): - _shutil.rmtree(fp) - removed += 1 - except Exception as e: - logger.debug("quarantine entry purge failed: %s", e) - _update_automation_progress(automation_id, - log_line=f'Removed {removed} quarantined items', log_type='success' if removed > 0 else 'info') - return {'status': 'completed', 'removed': str(removed)} - - def _auto_cleanup_wishlist(config): - automation_id = config.get('_automation_id') - db = get_database() - removed = db.remove_wishlist_duplicates(get_current_profile_id()) - _update_automation_progress(automation_id, - log_line=f'Removed {removed or 0} duplicate wishlist entries', log_type='success' if removed else 'info') - return {'status': 'completed', 'removed': str(removed or 0)} - - def _auto_update_discovery_pool(config): - automation_id = config.get('_automation_id') - try: - from core.watchlist_scanner import get_watchlist_scanner - scanner = get_watchlist_scanner(spotify_client) - _update_automation_progress(automation_id, - log_line='Updating discovery pool...', log_type='info') - scanner.update_discovery_pool_incremental(get_current_profile_id()) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line='Discovery pool updated', log_type='success') - return {'status': 'completed', '_manages_own_progress': True} - except Exception as e: - _update_automation_progress(automation_id, status='error', - phase='Error', log_line=str(e), log_type='error') - return {'status': 'error', 'reason': str(e), '_manages_own_progress': True} - - def _auto_start_quality_scan(config): - automation_id = config.get('_automation_id') - if quality_scanner_state.get('status') == 'running': - return {'status': 'skipped', 'reason': 'Quality scan already running'} - - scope = config.get('scope', 'watchlist') - # Pre-set status before submit so polling loop doesn't see stale 'finished' from last run - with quality_scanner_lock: - quality_scanner_state["status"] = "running" - quality_scanner_executor.submit(_run_quality_scanner, scope, get_current_profile_id()) - _update_automation_progress(automation_id, - log_line=f'Quality scan started (scope: {scope})', log_type='info') - - # Monitor quality scanner progress (max 2 hours) - time.sleep(1) # Brief pause for executor to start - poll_start = time.time() - while time.time() - poll_start < 7200: - time.sleep(3) - status = quality_scanner_state.get('status', 'idle') - if status not in ('running',): - break - phase = quality_scanner_state.get('phase', 'Scanning...') - progress = quality_scanner_state.get('progress', 0) - processed = quality_scanner_state.get('processed', 0) - total = quality_scanner_state.get('total', 0) - _update_automation_progress(automation_id, - phase=phase, progress=progress, - processed=processed, total=total) - else: - # 2-hour timeout reached - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Quality scan timed out after 2 hours', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - # Check actual exit status (could be 'finished' or 'error') - final_status = quality_scanner_state.get('status', 'idle') - if final_status == 'error': - err = quality_scanner_state.get('error_message', 'Unknown error') - _update_automation_progress(automation_id, status='error', progress=100, - phase='Error', log_line=err, log_type='error') - return {'status': 'error', 'reason': err, '_manages_own_progress': True} - - issues = quality_scanner_state.get('low_quality', 0) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line=f'Quality scan complete — {issues} issues found', log_type='success') - return { - 'status': 'completed', 'scope': scope, '_manages_own_progress': True, - 'tracks_scanned': quality_scanner_state.get('processed', 0), - 'quality_met': quality_scanner_state.get('quality_met', 0), - 'low_quality': issues, - 'matched': quality_scanner_state.get('matched', 0), - } - - def _auto_backup_database(config): - import sqlite3, glob as _glob - automation_id = config.get('_automation_id') - db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') - if not os.path.exists(db_path): - return {'status': 'error', 'reason': 'Database file not found'} - max_backups = 5 - timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') - backup_path = f"{db_path}.backup_{timestamp}" - # Use SQLite backup API for safe hot-copy of active database - src = sqlite3.connect(db_path) - dst = sqlite3.connect(backup_path) - src.backup(dst) - dst.close() - src.close() - size_mb = round(os.path.getsize(backup_path) / (1024 * 1024), 1) - # Rolling cleanup — keep only the newest N backups - existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime) - while len(existing) > max_backups: - try: - os.remove(existing.pop(0)) - except Exception as e: - logger.debug("rolling backup cleanup failed: %s", e) - _update_automation_progress(automation_id, - log_line=f'Backup created: {size_mb}MB ({os.path.basename(backup_path)})', log_type='success') - return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)} - - def _auto_refresh_beatport_cache(config): - """Refresh Beatport homepage cache by calling each endpoint internally.""" - automation_id = config.get('_automation_id') - sections = [ - ('hero_tracks', '/api/beatport/hero-tracks', 'Hero Tracks'), - ('new_releases', '/api/beatport/new-releases', 'New Releases'), - ('featured_charts', '/api/beatport/featured-charts', 'Featured Charts'), - ('dj_charts', '/api/beatport/dj-charts', 'DJ Charts'), - ('top_10_lists', '/api/beatport/homepage/top-10-lists', 'Top 10 Lists'), - ('top_10_releases', '/api/beatport/homepage/top-10-releases-cards', 'Top 10 Releases'), - ('hype_picks', '/api/beatport/hype-picks', 'Hype Picks'), - ] - # Invalidate all homepage cache timestamps so endpoints re-scrape - with beatport_data_cache['cache_lock']: - for key in beatport_data_cache['homepage']: - beatport_data_cache['homepage'][key]['timestamp'] = 0 - beatport_data_cache['homepage'][key]['data'] = None - - refreshed = 0 - errors = [] - with app.test_client() as client: - for idx, (_, endpoint, label) in enumerate(sections): - _update_automation_progress(automation_id, - progress=(idx / len(sections)) * 100, - phase=f'Scraping: {label}', - current_item=label) - try: - resp = client.get(endpoint) - if resp.status_code == 200: - refreshed += 1 - _update_automation_progress(automation_id, - log_line=f'{label}: cached', log_type='success') - else: - errors.append(label) - _update_automation_progress(automation_id, - log_line=f'{label}: HTTP {resp.status_code}', log_type='error') - except Exception as e: - errors.append(label) - _update_automation_progress(automation_id, - log_line=f'{label}: {str(e)}', log_type='error') - if idx < len(sections) - 1: - time.sleep(2) - - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line=f'Refreshed {refreshed}/{len(sections)} sections', log_type='success') - return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors)), - '_manages_own_progress': True} - - def _auto_clean_search_history(config): - """Remove old searches from Soulseek.""" - automation_id = config.get('_automation_id') - # Skip if soulseek is not the active download source or in hybrid order - dl_mode = config_manager.get('download_source.mode', 'hybrid') - hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) - soulseek_active = (dl_mode == 'soulseek' or - (dl_mode == 'hybrid' and 'soulseek' in hybrid_order)) - # Reach the underlying SoulseekClient via the orchestrator's - # generic accessor. - slskd = download_orchestrator.client('soulseek') if download_orchestrator else None - if not soulseek_active or not slskd or not slskd.base_url: - _update_automation_progress(automation_id, - log_line='Soulseek not active — skipped', log_type='skip') - return {'status': 'skipped'} - if not config_manager.get('soulseek.auto_clear_searches', True): - _update_automation_progress(automation_id, - log_line='Auto-clear disabled in settings', log_type='skip') - return {'status': 'skipped'} - try: - success = run_async(download_orchestrator.maintain_search_history_with_buffer( - keep_searches=50, trigger_threshold=200 - )) - if success: - _update_automation_progress(automation_id, - log_line='Search history maintenance completed', log_type='success') - return {'status': 'completed'} - else: - _update_automation_progress(automation_id, - log_line='No cleanup needed', log_type='skip') - return {'status': 'completed'} - except Exception as e: - return {'status': 'error', 'error': str(e)} - - def _auto_clean_completed_downloads(config): - """Clear completed downloads and empty directories.""" - automation_id = config.get('_automation_id') - try: - has_active_batches = False - has_post_processing = False - with tasks_lock: - for batch_data in download_batches.values(): - if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: - has_active_batches = True - break - if not has_active_batches: - for task_data in download_tasks.values(): - if task_data.get('status') == 'post_processing': - has_post_processing = True - break - - if has_active_batches: - _update_automation_progress(automation_id, - log_line='Skipped — downloads active', log_type='skip') - return {'status': 'completed'} - - run_async(download_orchestrator.clear_all_completed_downloads()) - if not has_post_processing: - _sweep_empty_download_directories() - _update_automation_progress(automation_id, - log_line='Download cleanup completed', log_type='success') - return {'status': 'completed'} - except Exception as e: - return {'status': 'error', 'reason': str(e)} - - def _auto_full_cleanup(config): - """Run all cleanup tasks: quarantine, download queue, empty dirs, staging, search history.""" - import shutil as _shutil - automation_id = config.get('_automation_id') - steps = [] - - # --- 1. Clear quarantine --- - _update_automation_progress(automation_id, phase='Clearing quarantine...', progress=0) - quarantine_path = os.path.join(docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), 'ss_quarantine') - q_removed = 0 - if os.path.exists(quarantine_path): - for f in os.listdir(quarantine_path): - fp = os.path.join(quarantine_path, f) - try: - if os.path.isfile(fp): - os.remove(fp) - q_removed += 1 - elif os.path.isdir(fp): - _shutil.rmtree(fp) - q_removed += 1 - except Exception as e: - logger.debug("quarantine entry purge failed: %s", e) - steps.append(f'Quarantine: removed {q_removed} items') - _update_automation_progress(automation_id, - log_line=f'Quarantine: removed {q_removed} items', log_type='success' if q_removed else 'info') - - # --- 2. Clear completed/errored/cancelled downloads from Soulseek queue --- - _update_automation_progress(automation_id, phase='Clearing download queue...', progress=20) - has_active_batches = False - has_post_processing = False - with tasks_lock: - for batch_data in download_batches.values(): - if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: - has_active_batches = True - break - if not has_active_batches: - for task_data in download_tasks.values(): - if task_data.get('status') == 'post_processing': - has_post_processing = True - break - if has_active_batches: - steps.append('Download queue: skipped (active batches)') - _update_automation_progress(automation_id, - log_line='Download queue: skipped (active batches)', log_type='skip') - else: - try: - run_async(download_orchestrator.clear_all_completed_downloads()) - steps.append('Download queue: cleared') - _update_automation_progress(automation_id, - log_line='Download queue: cleared', log_type='success') - except Exception as e: - steps.append(f'Download queue: error ({e})') - _update_automation_progress(automation_id, - log_line=f'Download queue: error ({e})', log_type='error') - - # --- 3. Sweep empty download directories --- - _update_automation_progress(automation_id, phase='Sweeping empty directories...', progress=40) - if has_active_batches or has_post_processing: - reason = 'active batches' if has_active_batches else 'post-processing active' - steps.append(f'Empty directories: skipped ({reason})') - _update_automation_progress(automation_id, - log_line=f'Empty directories: skipped ({reason})', log_type='skip') - else: - dirs_removed = _sweep_empty_download_directories() - steps.append(f'Empty directories: removed {dirs_removed}') - _update_automation_progress(automation_id, - log_line=f'Empty directories: removed {dirs_removed}', log_type='success' if dirs_removed else 'info') - - # --- 4. Sweep empty staging directories --- - _update_automation_progress(automation_id, phase='Sweeping import folder...', progress=60) - staging_path = get_staging_path() - s_removed = 0 - if os.path.isdir(staging_path): - for dirpath, _dirnames, _filenames in os.walk(staging_path, topdown=False): - if os.path.normpath(dirpath) == os.path.normpath(staging_path): - continue - try: - entries = os.listdir(dirpath) - except OSError: - continue - visible = [e for e in entries if not e.startswith('.')] - if not visible: - for hidden in entries: - try: - os.remove(os.path.join(dirpath, hidden)) - except Exception as e: - logger.debug("hidden file cleanup failed: %s", e) - try: - os.rmdir(dirpath) - s_removed += 1 - except OSError: - pass - steps.append(f'Staging: removed {s_removed} empty directories') - _update_automation_progress(automation_id, - log_line=f'Staging: removed {s_removed} empty directories', log_type='success' if s_removed else 'info') - - # --- 5. Clean search history (if enabled) --- - _update_automation_progress(automation_id, phase='Cleaning search history...', progress=80) - try: - if not config_manager.get('soulseek.auto_clear_searches', True): - steps.append('Search cleanup: disabled in settings') - _update_automation_progress(automation_id, - log_line='Search cleanup: disabled in settings', log_type='skip') - else: - run_async(download_orchestrator.maintain_search_history_with_buffer( - keep_searches=50, trigger_threshold=200 - )) - steps.append('Search history: cleaned') - _update_automation_progress(automation_id, - log_line='Search history: cleaned', log_type='success') - except Exception as e: - steps.append(f'Search history: error ({e})') - _update_automation_progress(automation_id, - log_line=f'Search history: error ({e})', log_type='error') - - total_removed = q_removed + s_removed - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line=f'Full cleanup complete — {total_removed} items removed', log_type='success') - return { - 'status': 'completed', - 'quarantine_removed': str(q_removed), - 'staging_removed': str(s_removed), - 'total_removed': str(total_removed), - 'steps': steps, - '_manages_own_progress': True, - } - - def _auto_run_script(config): - """Execute a user script from the scripts directory.""" - import subprocess as _sp - script_name = config.get('script_name', '') - timeout = min(int(config.get('timeout', 60)), 300) - automation_id = config.get('_automation_id') - - if not script_name: - return {'status': 'error', 'error': 'No script selected'} - - scripts_dir = docker_resolve_path(config_manager.get('scripts.path', './scripts')) - if not scripts_dir or not os.path.isdir(scripts_dir): - os.makedirs(scripts_dir, exist_ok=True) - return {'status': 'error', 'error': 'Scripts directory is empty. Add scripts to the scripts/ folder.'} - - script_path = os.path.join(scripts_dir, script_name) - script_path = os.path.realpath(script_path) - - # Security: block path traversal - if not script_path.startswith(os.path.realpath(scripts_dir)): - return {'status': 'error', 'error': 'Script path traversal blocked'} - - if not os.path.isfile(script_path): - return {'status': 'error', 'error': f'Script not found: {script_name}'} - - _update_automation_progress(automation_id, phase=f'Running {script_name}...', progress=10) - - # Build environment with SoulSync context - env = os.environ.copy() - event_data = config.get('_event_data') or {} - env['SOULSYNC_EVENT'] = str(event_data.get('type', '')) - env['SOULSYNC_AUTOMATION'] = config.get('_automation_name', '') - env['SOULSYNC_SCRIPTS_DIR'] = scripts_dir - - try: - # Determine how to run the script - if script_path.endswith('.py'): - cmd = ['python', script_path] - elif script_path.endswith('.sh'): - cmd = ['bash', script_path] - else: - cmd = [script_path] - - result = _sp.run( - cmd, - capture_output=True, text=True, timeout=timeout, - cwd=scripts_dir, env=env - ) - - _update_automation_progress(automation_id, phase='Script completed', progress=100) - - stdout = result.stdout[:2000] if result.stdout else '' - stderr = result.stderr[:1000] if result.stderr else '' - - if result.returncode == 0: - logger.info(f"Script '{script_name}' completed (exit 0)") - else: - logger.warning(f"Script '{script_name}' exited with code {result.returncode}") - - return { - 'status': 'completed' if result.returncode == 0 else 'error', - 'exit_code': str(result.returncode), - 'stdout': stdout, - 'stderr': stderr, - 'script': script_name, - } - except _sp.TimeoutExpired: - _update_automation_progress(automation_id, phase='Script timed out', progress=100) - return {'status': 'error', 'error': f'Script timed out after {timeout}s', 'script': script_name} - except Exception as e: - return {'status': 'error', 'error': str(e), 'script': script_name} - - automation_engine.register_action_handler('run_script', _auto_run_script) - automation_engine.register_action_handler('full_cleanup', _auto_full_cleanup) - - automation_engine.register_action_handler('start_database_update', _auto_start_database_update, - lambda: db_update_state.get('status') == 'running') - automation_engine.register_action_handler('deep_scan_library', _auto_deep_scan_library, - lambda: db_update_state.get('status') == 'running') - automation_engine.register_action_handler('run_duplicate_cleaner', _auto_run_duplicate_cleaner, - lambda: duplicate_cleaner_state.get('status') == 'running') - automation_engine.register_action_handler('clear_quarantine', _auto_clear_quarantine) - automation_engine.register_action_handler('cleanup_wishlist', _auto_cleanup_wishlist) - automation_engine.register_action_handler('update_discovery_pool', _auto_update_discovery_pool) - automation_engine.register_action_handler('start_quality_scan', _auto_start_quality_scan, - lambda: quality_scanner_state.get('status') == 'running') - automation_engine.register_action_handler('backup_database', _auto_backup_database) - automation_engine.register_action_handler('refresh_beatport_cache', _auto_refresh_beatport_cache) - automation_engine.register_action_handler('clean_search_history', _auto_clean_search_history) - automation_engine.register_action_handler('clean_completed_downloads', _auto_clean_completed_downloads) - - def _auto_search_and_download(config): - """Search for a track and download the best match.""" - automation_id = config.get('_automation_id') - query = config.get('query', '').strip() - # Event-triggered: pull query from event data (e.g. webhook_received) - if not query: - event_data = config.get('_event_data', {}) - query = (event_data.get('query', '') or '').strip() - if not query: - if automation_id: - _update_automation_progress(automation_id, - log_line='No search query provided', log_type='error') - return {'status': 'error', 'error': 'No search query provided'} - try: - if automation_id: - _update_automation_progress(automation_id, - phase='Searching', log_line=f'Searching: {query}', log_type='info') - result = run_async(download_orchestrator.search_and_download_best(query)) - if result: - if automation_id: - _update_automation_progress(automation_id, - log_line=f'Download started for: {query}', log_type='success') - return {'status': 'completed', 'query': query, 'download_id': result} - else: - if automation_id: - _update_automation_progress(automation_id, - log_line=f'No match found for: {query}', log_type='warning') - return {'status': 'not_found', 'query': query, 'error': 'No match found'} - except Exception as e: - if automation_id: - _update_automation_progress(automation_id, - log_line=f'Error: {e}', log_type='error') - return {'status': 'error', 'query': query, 'error': str(e)} - - automation_engine.register_action_handler('search_and_download', _auto_search_and_download) - - # Register progress tracking callbacks - def _progress_init(aid, name, action_type): - _init_automation_progress(aid, name, action_type) - - def _progress_finish(aid, result): - result_status = result.get('status', '') - # Skip for handlers that manage their own progress lifecycle - # (they call _update_automation_progress(status='finished') themselves) - if result.get('_manages_own_progress'): - return - status = 'error' if result_status == 'error' else 'finished' - msg = result.get('error', result.get('reason', result_status or 'done')) - _update_automation_progress(aid, status=status, progress=100, - phase='Error' if status == 'error' else 'Complete', - log_line=msg, log_type='error' if status == 'error' else 'success') - - def _record_automation_history(aid, result): - """Capture progress state into run history before cleanup clears it.""" - _auto_progress.record_history(aid, result, get_database()) - - automation_engine.register_progress_callbacks(_progress_init, _progress_finish, _update_automation_progress, _record_automation_history) - - # Register permanent callback: when any scan completes, emit library_scan_completed event - # This replaces the hardcoded scan_completion_callback → trigger_automatic_database_update chain - if web_scan_manager: - def _on_library_scan_completed(): - if automation_engine: - server_type = getattr(web_scan_manager, '_current_server_type', None) or 'unknown' - automation_engine.emit('library_scan_completed', { - 'server_type': server_type, - }) - web_scan_manager.add_scan_completion_callback(_on_library_scan_completed) logger.info("Automation action handlers registered") @@ -2985,6 +1586,7 @@ def _shutdown_runtime_components(): (import_singles_executor, "import singles executor"), (tidal_discovery_executor, "tidal discovery executor"), (deezer_discovery_executor, "deezer discovery executor"), + (qobuz_discovery_executor, "qobuz discovery executor"), (spotify_public_discovery_executor, "spotify public discovery executor"), (youtube_discovery_executor, "youtube discovery executor"), (beatport_discovery_executor, "beatport discovery executor"), @@ -3150,21 +1752,22 @@ def _find_downloaded_file(download_path, track_data): audio_extensions = {'.mp3', '.flac', '.ogg', '.aac', '.wma', '.wav', '.m4a'} target_filename = extract_filename(track_data.get('filename', '')) - # YOUTUBE/TIDAL/QOBUZ/HIFI SUPPORT: Handle encoded filename format "id||title" + # YOUTUBE/TIDAL/QOBUZ/HIFI/AMAZON SUPPORT: Handle encoded filename format "id||title" # The file on disk will be "title.ext", not "id||title" is_youtube = track_data.get('username') == 'youtube' is_tidal = track_data.get('username') == 'tidal' is_qobuz = track_data.get('username') == 'qobuz' is_hifi = track_data.get('username') == 'hifi' - is_streaming_source = is_youtube or is_tidal or is_qobuz or is_hifi + is_amazon = track_data.get('username') == 'amazon' + is_streaming_source = is_youtube or is_tidal or is_qobuz or is_hifi or is_amazon target_filename_youtube = None if is_streaming_source and '||' in target_filename: _, title = target_filename.split('||', 1) - if is_tidal or is_qobuz or is_hifi: - # Tidal/Qobuz/HiFi files can be flac or m4a — match any audio extension + if is_tidal or is_qobuz or is_hifi or is_amazon: + # Tidal/Qobuz/HiFi/Amazon files can be flac, opus, eac3, or m4a — match any audio extension safe_title = re.sub(r'[<>:"/\\|?*]', '_', title) target_filename_youtube = safe_title # Extension-less for flexible matching - source_name = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else 'Tidal') + source_name = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else ('Amazon' if is_amazon else 'Tidal')) logger.debug(f"[{source_name} Stream] Looking for file starting with: {target_filename_youtube}") else: # yt-dlp will create "Title.mp3" from "Title" @@ -3202,11 +1805,11 @@ def _find_downloaded_file(download_path, track_data): # For Tidal, compare without extension (file could be .flac or .m4a) compare_target = target_filename_youtube.lower() compare_file = file.lower() - if is_tidal or is_qobuz or is_hifi: + if is_tidal or is_qobuz or is_hifi or is_amazon: compare_file = os.path.splitext(compare_file)[0] similarity = SequenceMatcher(None, compare_file, compare_target).ratio() - source_label = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else ('Tidal' if is_tidal else 'YouTube')) + source_label = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else ('Amazon' if is_amazon else ('Tidal' if is_tidal else 'YouTube'))) logger.debug(f"[{source_label} Stream] Comparing: '{file}' vs '{target_filename_youtube}' = {similarity:.2f}") # Keep track of best match @@ -3226,7 +1829,7 @@ def _find_downloaded_file(download_path, track_data): # For YouTube/Tidal, if we found a good enough match (80%+), use it if is_streaming_source and best_match and best_similarity >= 0.80: - source_label = 'Qobuz' if is_qobuz else ('Tidal' if is_tidal else 'YouTube') + source_label = 'Qobuz' if is_qobuz else ('Amazon' if is_amazon else ('Tidal' if is_tidal else 'YouTube')) logger.debug(f"Found good match ({best_similarity:.2f}) for {source_label} streaming file: {best_match}") return best_match @@ -3257,6 +1860,8 @@ SERVICE_CONFIG_REGISTRY = { 'spotify': {'required': ['client_id', 'client_secret']}, 'itunes': {'always': True}, # default storefront works anon 'deezer': {'always': True}, # anon search works, premium ARL is optional + 'musicbrainz': {'always': True}, # public API, no credentials required + 'amazon': {'always': True}, # T2Tunes proxy, no credentials required 'discogs': {'required': ['token']}, 'tidal': {'custom': lambda _svc: _tidal_has_auth_token()}, 'qobuz': {'any_of': [['email', 'password'], ['token'], ['user_auth_token']]}, @@ -3402,6 +2007,7 @@ def _get_enrichment_status(): ('genius', 'Genius', lambda: genius_worker), ('audiodb', 'AudioDB', lambda: audiodb_worker), ('discogs', 'Discogs', lambda: discogs_worker), + ('amazon_enrichment', 'Amazon Music', lambda: amazon_worker), ] # Config-based "configured" checks for services that need API keys/credentials @@ -3510,10 +2116,15 @@ def get_status(): _status_cache_timestamps['media_server'] = current_time # else: use cached value - # Test Soulseek - only if it's the active source or in the hybrid order - if current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL: - download_mode = config_manager.get('download_source.mode', 'hybrid') - hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + download_mode = config_manager.get('download_source.mode', 'hybrid') + hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) + if isinstance(hybrid_order, str): + hybrid_order = [hybrid_order] + source_cache_key = f"{download_mode}:{','.join(str(s) for s in (hybrid_order or []))}" + + # Test Soulseek/download source only when the cached source selection is still current. + if (current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL or + _status_cache['soulseek'].get('source_cache_key') != source_cache_key): soulseek_relevant = (download_mode == 'soulseek' or (download_mode == 'hybrid' and 'soulseek' in hybrid_order)) @@ -3521,15 +2132,28 @@ def get_status(): # don't depend on slskd being reachable — when one of these is the # active source, surface "connected" without probing slskd so the # dashboard / sidebar indicator stays green. - serverless_sources = ('youtube', 'hifi', 'qobuz', 'tidal', 'deezer_dl', 'lidarr', 'soundcloud') + serverless_sources = ('youtube', 'hifi', 'qobuz', 'tidal', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon') is_serverless = (download_mode in serverless_sources or (download_mode == 'hybrid' and hybrid_order and any(s in serverless_sources for s in hybrid_order))) + external_client_sources = ('torrent', 'usenet') + external_client_relevant = ( + download_mode in external_client_sources or + (download_mode == 'hybrid' and + hybrid_order and any(s in external_client_sources for s in hybrid_order)) + ) # Serverless check first — avoids slow slskd timeout when YouTube/HiFi are in hybrid order if is_serverless: soulseek_status = True soulseek_response_time = 0 + elif external_client_relevant and download_orchestrator: + soulseek_start = time.time() + try: + soulseek_status = run_async(download_orchestrator.check_connection()) + except Exception: + soulseek_status = False + soulseek_response_time = (time.time() - soulseek_start) * 1000 elif soulseek_relevant and download_orchestrator: soulseek_start = time.time() try: @@ -3543,13 +2167,17 @@ def get_status(): _status_cache['soulseek'] = { 'connected': soulseek_status, - 'response_time': round(soulseek_response_time, 1) + 'response_time': round(soulseek_response_time, 1), + 'source_cache_key': source_cache_key, } _status_cache_timestamps['soulseek'] = current_time # Include download source mode so frontend can update labels - download_mode = config_manager.get('download_source.mode', 'hybrid') _status_cache['soulseek']['source'] = download_mode + soulseek_data = { + key: value for key, value in _status_cache['soulseek'].items() + if key != 'source_cache_key' + } # Count active downloads for nav badge active_dl_count = 0 @@ -3562,7 +2190,7 @@ def get_status(): 'metadata_source': metadata_status['metadata_source'], 'spotify': metadata_status['spotify'], 'media_server': _status_cache['media_server'], - 'soulseek': _status_cache['soulseek'], + 'soulseek': soulseek_data, 'active_media_server': active_server, 'enrichment': _get_enrichment_status(), 'active_downloads': active_dl_count, @@ -4148,7 +2776,7 @@ def handle_settings(): if 'active_media_server' in new_settings: config_manager.set_active_media_server(new_settings['active_media_server']) - for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'qobuz', 'hifi_download', 'deezer_download', 'lidarr_download', 'listenbrainz', 'acoustid', 'lastfm', 'genius', 'import', 'lossy_copy', 'listening_stats', 'ui_appearance', 'youtube', 'content_filter', 'itunes', 'm3u_export', 'musicbrainz', 'deezer', 'audiodb', 'metadata', 'hydrabase', 'security', 'discogs', 'library', 'discover', 'wishlist', 'genre_whitelist', 'post_processing']: + for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'qobuz', 'hifi_download', 'deezer_download', 'amazon_download', 'lidarr_download', 'prowlarr', 'torrent_client', 'usenet_client', 'listenbrainz', 'acoustid', 'lastfm', 'genius', 'import', 'lossy_copy', 'listening_stats', 'ui_appearance', 'youtube', 'content_filter', 'itunes', 'm3u_export', 'musicbrainz', 'deezer', 'audiodb', 'metadata', 'hydrabase', 'security', 'discogs', 'library', 'discover', 'wishlist', 'genre_whitelist', 'post_processing']: if service in new_settings: for key, value in new_settings[service].items(): config_manager.set(f'{service}.{key}', value) @@ -4438,6 +3066,36 @@ def hydrabase_send(): _hydrabase_ws = None return jsonify({"success": False, "error": str(e)}), 500 +@app.route('/api/prowlarr/indexers', methods=['GET']) +def prowlarr_indexers_endpoint(): + """List indexers Prowlarr currently exposes — name, protocol, enabled state. + + Drives the Indexers panel on Settings → Indexers & Downloaders so + the user can see what they're searching against without leaving + SoulSync. Returns ``[]`` if Prowlarr isn't configured / reachable. + """ + try: + from core.prowlarr_client import ProwlarrClient + client = ProwlarrClient() + if not client.is_configured(): + return jsonify({"success": False, "error": "Prowlarr not configured", "indexers": []}), 200 + indexers = run_async(client.get_indexers()) + items = [ + { + 'id': idx.id, + 'name': idx.name, + 'protocol': idx.protocol, + 'enable': idx.enable, + 'privacy': idx.privacy, + } + for idx in indexers + ] + return jsonify({"success": True, "indexers": items}) + except Exception as e: + logger.error(f"prowlarr indexers fetch error: {e}") + return jsonify({"success": False, "error": str(e), "indexers": []}), 500 + + @app.route('/api/settings/log-level', methods=['GET', 'POST']) @admin_only def handle_log_level(): @@ -7196,7 +5854,7 @@ def start_download(): if download_id: # Register download for post-processing (simple transfer to /Transfer) context_key = _make_context_key(username, filename) - is_streaming_source = username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') + is_streaming_source = username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon') with matched_context_lock: matched_downloads_context[context_key] = { 'search_result': { @@ -7252,6 +5910,14 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): from difflib import SequenceMatcher from unidecode import unidecode + audio_extensions = { + '.mp3', '.flac', '.m4a', '.aac', '.ogg', '.opus', '.wav', '.wma', '.alac', + '.aiff', '.aif', '.dsf', '.dff', '.ape', + } + + def _is_audio_candidate(path): + return os.path.splitext(str(path or ''))[1].lower() in audio_extensions + # YOUTUBE/TIDAL SUPPORT: Handle encoded filename format "id||title" # Extract just the title part for file matching if '||' in api_filename: @@ -7286,9 +5952,12 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): # Skip quarantine folder — contains known-wrong files from AcoustID verification dirs[:] = [d for d in dirs if d != 'ss_quarantine'] for file in files: + file_path = os.path.join(root, file) + if not _is_audio_candidate(file_path): + continue + # Direct basename match if os.path.basename(file) == target_basename: - file_path = os.path.join(root, file) # Fast path: if path aligns with expected directory structure, return now if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed match in {location_name}: {file_path}") @@ -7305,7 +5974,6 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): file_stem, file_ext_part = os.path.splitext(file) stripped_stem = re.sub(r'_\d{10,}$', '', file_stem) if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename: - file_path = os.path.join(root, file) if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed dedup match in {location_name}: {file_path}") return file_path, 1.0 @@ -7321,7 +5989,7 @@ def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): if similarity > highest_fuzzy_similarity: highest_fuzzy_similarity = similarity - best_fuzzy_path = os.path.join(root, file) + best_fuzzy_path = file_path # Return best exact match (disambiguated by path), or fall back to fuzzy if exact_matches: @@ -7463,6 +6131,10 @@ def get_download_status(): transfer_id = file_info.get('id') if transfer_id: try: + logger.info( + f"[CancelTrigger:web.orphan_cleanup] " + f"download_id={transfer_id} username={username}" + ) run_async(download_orchestrator.cancel_download(str(transfer_id), username, remove=True)) except Exception as e: logger.debug("orphan transfer cancel failed: %s", e) @@ -7578,7 +6250,7 @@ def get_download_status(): all_streaming_downloads = run_async(download_orchestrator.get_all_downloads()) for download in all_streaming_downloads: - if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): + if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): source_label = download.username.title() # Convert DownloadStatus to transfer format that frontend expects streaming_transfer = { @@ -8195,6 +6867,94 @@ def clear_quarantine(): logger.error(f"[Quarantine] Error clearing quarantine: {e}") return jsonify({"success": False, "error": str(e)}), 500 + +def _get_quarantine_dir(): + return os.path.join( + docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), + 'ss_quarantine', + ) + + +@app.route('/api/quarantine/list', methods=['GET']) +def list_quarantine(): + """Return all quarantined files with sidecar metadata.""" + try: + from core.imports.quarantine import list_quarantine_entries + entries = list_quarantine_entries(_get_quarantine_dir()) + return jsonify({"success": True, "entries": entries}) + except Exception as e: + logger.error(f"[Quarantine] Error listing entries: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/quarantine/', methods=['DELETE']) +def delete_quarantine_item(entry_id): + """Delete a single quarantined file + sidecar.""" + try: + from core.imports.quarantine import delete_quarantine_entry + ok = delete_quarantine_entry(_get_quarantine_dir(), entry_id) + if not ok: + return jsonify({"success": False, "error": "Entry not found"}), 404 + return jsonify({"success": True}) + except Exception as e: + logger.error(f"[Quarantine] Error deleting {entry_id}: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/quarantine//approve', methods=['POST']) +def approve_quarantine_item(entry_id): + """One-click approve: restore the file and re-run post-process with the + quarantine gates skipped for this explicit user-approved pass.""" + try: + from core.imports.quarantine import approve_quarantine_entry + # Restore inside the soulseek download dir so existing path-resolution + # logic finds it. Unique subdir keeps it from re-mingling with active + # transfers. + restore_dir = os.path.join( + docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), + 'Transfer', + ) + result = approve_quarantine_entry(_get_quarantine_dir(), entry_id, restore_dir) + if result is None: + return jsonify({ + "success": False, + "error": "Cannot one-click approve — entry has thin sidecar (no embedded context). Use 'Recover to Staging' instead.", + }), 400 + restored_path, context, trigger = result + # User approval means "import this file"; skip all quarantine gates + # for this one restored pass so multi-reason failures do not loop. + context['_skip_quarantine_check'] = 'all' + context['_approved_quarantine_trigger'] = trigger + # Re-dispatch through the same pipeline. Run async so the HTTP + # request returns quickly — UI polls /list to see the entry vanish. + context_key = f"approve_{entry_id}_{int(time.time())}" + threading.Thread( + target=lambda: _post_process_matched_download(context_key, context, restored_path), + daemon=True, + ).start() + logger.info(f"[Quarantine] Approved {entry_id} (original_trigger={trigger}, bypass=all) → re-running pipeline") + return jsonify({"success": True, "trigger_bypassed": "all", "original_trigger": trigger}) + except Exception as e: + logger.error(f"[Quarantine] Error approving {entry_id}: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/quarantine//recover', methods=['POST']) +def recover_quarantine_item(entry_id): + """Fallback for legacy thin sidecars: move file into Staging so the user + can manually finish via the existing Import flow.""" + try: + from core.imports.quarantine import recover_to_staging + from core.imports.staging import get_staging_path + target = recover_to_staging(_get_quarantine_dir(), get_staging_path(), entry_id) + if not target: + return jsonify({"success": False, "error": "Entry not found"}), 404 + return jsonify({"success": True, "staged_path": target}) + except Exception as e: + logger.error(f"[Quarantine] Error recovering {entry_id}: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + @app.route('/api/scan/request', methods=['POST']) def request_media_scan(): """ @@ -8594,6 +7354,13 @@ def _build_source_only_artist_detail(artist_id, artist_name, source): except Exception as e: logger.debug(f"Discogs client resolution failed: {e}") + az = None + try: + from core.metadata.registry import get_amazon_client + az = get_amazon_client() + except Exception as e: + logger.debug(f"Amazon client resolution failed: {e}") + try: lastfm_api_key = config_manager.get('lastfm.api_key', '') or None except Exception: @@ -8607,6 +7374,7 @@ def _build_source_only_artist_detail(artist_id, artist_name, source): deezer_client=dz, itunes_client=it, discogs_client=dc, + amazon_client=az, lastfm_api_key=lastfm_api_key, ) return jsonify(payload), status @@ -8675,17 +7443,17 @@ def get_artist_detail(artist_id): logger.info(f"Found artist: {artist_info['name']} with {len(owned_releases['albums'])} albums") - # Fix artist image URL - logger.info(f"Artist image before fix: '{artist_info.get('image_url')}'") + # Fix artist image URL. + # NOTE: don't log image_url or the full artist_info dict here. + # The fixed URL embeds the media-server token (and the proxy + # variant URL-encodes it), so logging at INFO writes the token + # straight into app.log. Issue: tokens leaked to disk on every + # artist-page render until this was scrubbed. if artist_info.get('image_url'): artist_info['image_url'] = fix_artist_image_url(artist_info['image_url']) - logger.info(f"Artist image after fix: '{artist_info['image_url']}'") else: logger.warning(f"No artist image URL found for {artist_info['name']}") - # Debug final artist data being sent - logger.info(f"Final artist data being sent: {artist_info}") - # Fix image URLs for all albums for album in owned_releases['albums']: if album.get('image_url'): @@ -8712,6 +7480,7 @@ def get_artist_detail(artist_id): 'itunes': artist_info.get('itunes_artist_id'), 'discogs': artist_info.get('discogs_id'), 'hydrabase': artist_info.get('soul_id'), + 'amazon': artist_info.get('amazon_id'), } artist_detail_discography = _get_artist_detail_discography( @@ -9970,6 +8739,9 @@ def library_check_tracks(): file_ext = os.path.splitext(matched_db_track.file_path or '')[1].lstrip('.').upper() or None owned_map[track_name] = { "owned": True, + "track_id": getattr(matched_db_track, 'id', None), + "title": getattr(matched_db_track, 'title', track_name), + "file_path": getattr(matched_db_track, 'file_path', None), "format": file_ext, "bitrate": matched_db_track.bitrate, "album": getattr(matched_db_track, 'album_title', None) @@ -11009,12 +9781,20 @@ def reorganize_album_preview(album_id): the apply endpoint, so the preview is guaranteed to match what apply would actually produce. - Optional body param ``source``: when provided, only that metadata - source is queried (no fallback chain).""" + Optional body params: + source: when provided, only that metadata source is queried + (no fallback chain). + mode: 'api' (default — query metadata source) or 'tags' (read + embedded file tags as the source of truth, issue #592).""" try: from core.library_reorganize import preview_album_reorganize data = request.get_json() or {} chosen_source = data.get('source') or None + metadata_source = data.get('mode') or config_manager.get( + 'library.reorganize_metadata_source', 'api' + ) or 'api' + if metadata_source not in ('api', 'tags'): + metadata_source = 'api' transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) result = preview_album_reorganize( album_id=album_id, @@ -11024,6 +9804,7 @@ def reorganize_album_preview(album_id): build_final_path_fn=_build_final_path_for_track, primary_source=chosen_source, strict_source=bool(chosen_source), + metadata_source=metadata_source, ) if result.get('status') == 'no_album': return jsonify({"success": False, "error": "Album not found"}), 404 @@ -11046,11 +9827,21 @@ def reorganize_album_files(album_id): source (optional): per-album source pick (Spotify / iTunes / Deezer / Discogs / Hydrabase). When omitted, the orchestrator uses the configured primary with fallback. + mode (optional): 'api' (default — query metadata source) or + 'tags' (read embedded file tags as the source of truth, + issue #592). When omitted, falls back to the + ``library.reorganize_metadata_source`` config setting, + then to 'api'. """ try: from core.reorganize_queue import get_queue data = request.get_json() or {} chosen_source = data.get('source') or None + metadata_source = data.get('mode') or config_manager.get( + 'library.reorganize_metadata_source', 'api' + ) or 'api' + if metadata_source not in ('api', 'tags'): + metadata_source = 'api' # Capture display fields at enqueue time so the status panel # can render them without a DB lookup later. @@ -11064,6 +9855,7 @@ def reorganize_album_files(album_id): artist_id=meta['artist_id'], artist_name=meta['artist_name'], source=chosen_source, + metadata_source=metadata_source, ) return jsonify({"success": True, **result}) except Exception as e: @@ -11081,20 +9873,29 @@ def reorganize_all_artist_albums(artist_id): source (optional): same pick applied to every album. Per-album overrides aren't supported here — use the per-album modal for that. + mode (optional): 'api' or 'tags' applied to every album, same + shape as the per-album endpoint. """ try: from core.reorganize_queue import get_queue data = request.get_json() or {} chosen_source = data.get('source') or None + metadata_source = data.get('mode') or config_manager.get( + 'library.reorganize_metadata_source', 'api' + ) or 'api' + if metadata_source not in ('api', 'tags'): + metadata_source = 'api' albums = get_database().get_artist_albums_for_reorganize(artist_id) if not albums: return jsonify({"success": False, "error": "No albums found for this artist"}), 404 - # Apply the user's chosen source to every album, then hand off - # to the queue's bulk-enqueue helper which owns the loop+tally. + # Apply the user's chosen source + mode to every album, then + # hand off to the queue's bulk-enqueue helper which owns the + # loop+tally. for album in albums: album['source'] = chosen_source + album['metadata_source'] = metadata_source result = get_queue().enqueue_many(albums) return jsonify({ @@ -11926,6 +10727,7 @@ _SERVICE_ID_COLUMNS = { 'genius': {'artist': 'genius_id', 'track': 'genius_id'}, 'tidal': {'artist': 'tidal_id', 'album': 'tidal_id', 'track': 'tidal_id'}, 'qobuz': {'artist': 'qobuz_id', 'album': 'qobuz_id', 'track': 'qobuz_id'}, + 'amazon': {'artist': 'amazon_id', 'album': 'amazon_id', 'track': 'amazon_id'}, } @app.route('/api/library/manual-match', methods=['PUT']) @@ -12049,6 +10851,57 @@ def library_clear_match(): return jsonify({"success": False, "error": str(e)}), 500 +@app.route('/api/library/album//import-existing-track', methods=['POST']) +@app.route('/api/library/album//missing-track/import-existing', methods=['POST']) +def library_import_existing_track_for_missing_slot(album_id): + """Use an existing library file as source audio for a missing album slot. + + The selected source file is copied to staging, then routed through the + normal post-processing pipeline with the target album/track metadata. The + original source file is never moved or deleted. + """ + try: + from core.library.missing_track_import import ( + MissingTrackImportDeps, + MissingTrackImportError, + import_existing_track_for_album_slot, + ) + + data = request.get_json() or {} + database = get_database() + deps = MissingTrackImportDeps( + database=database, + config_manager=config_manager, + post_process_fn=_post_process_matched_download, + resolve_library_file_path_fn=_resolve_library_file_path, + docker_resolve_path_fn=docker_resolve_path, + sync_tracks_to_server_fn=_sync_tracks_to_server, + service_id_columns=_SERVICE_ID_COLUMNS, + ) + + result = import_existing_track_for_album_slot(album_id, data, deps) + updated = database.get_artist_full_detail(result.get('artist_id')) + if updated.get('success'): + if updated.get('artist', {}).get('thumb_url'): + updated['artist']['thumb_url'] = fix_artist_image_url(updated['artist']['thumb_url']) + for album in updated.get('albums', []): + if album.get('thumb_url'): + album['thumb_url'] = fix_artist_image_url(album['thumb_url']) + + return jsonify({ + "success": True, + "message": "Imported existing track into album", + "track_id": result.get('track_id'), + "final_path": result.get('final_path'), + "updated_data": updated if updated.get('success') else None, + }) + except MissingTrackImportError as e: + return jsonify({"success": False, "error": str(e)}), e.status_code + except Exception as e: + logger.error(f"Error importing existing track for album slot: {e}", exc_info=True) + return jsonify({"success": False, "error": str(e)}), 500 + + @app.route('/api/library/track/', methods=['DELETE']) def library_delete_track(track_id): """Delete a track from the database, optionally deleting the file and blacklisting the source.""" @@ -12389,7 +11242,7 @@ def redownload_search_sources(track_id): quality = ext if ext in ('FLAC', 'MP3', 'OPUS', 'OGG', 'M4A', 'WAV') else candidate.quality or '' svc = source_name if source_name != 'default' else 'hybrid' uname = candidate.username - if uname in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): + if uname in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): svc = uname source_candidates.append({ 'username': uname, @@ -13287,7 +12140,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar logger.info(f"Processing enhanced album download for '{spotify_album['name']}' with {len(enhanced_tracks)} matched tracks") # Compute total_discs for multi-disc album subfolder support - total_discs = max((t['spotify_track'].get('disc_number', 1) for t in enhanced_tracks), default=1) + total_discs = max((t['spotify_track'].get('disc_number') or 1 for t in enhanced_tracks), default=1) spotify_album['total_discs'] = total_discs started_count = 0 @@ -13429,7 +12282,7 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): # Compute total_discs for multi-disc album subfolder support if official_spotify_tracks: - total_discs = max((t.get('disc_number', 1) for t in official_spotify_tracks), default=1) + total_discs = max((t.get('disc_number') or 1 for t in official_spotify_tracks), default=1) else: total_discs = 1 spotify_album['total_discs'] = total_discs @@ -15078,6 +13931,8 @@ def _build_retag_deps(): global retag_state retag_state = value + from core.metadata.lyrics import generate_lrc_file as _generate_lrc_file + return _library_retag.RetagDeps( config_manager=config_manager, retag_lock=retag_lock, @@ -15092,6 +13947,7 @@ def _build_retag_deps(): _get_retag_state=_get_state, _set_retag_state=_set_state, get_database=_get_db, + generate_lrc_file=_generate_lrc_file, ) @@ -15483,7 +14339,7 @@ def _pause_workers_for_scan(): 'mb': mb_worker, 'spotify': spotify_enrichment_worker, 'itunes': itunes_enrichment_worker, 'deezer': deezer_worker, 'audiodb': audiodb_worker, 'discogs': discogs_worker, 'lastfm': lastfm_worker, 'genius': genius_worker, 'tidal': tidal_enrichment_worker, 'qobuz': qobuz_enrichment_worker, - 'repair': repair_worker, 'soulid': soulid_worker, + 'amazon': amazon_worker, 'repair': repair_worker, 'soulid': soulid_worker, } for name, w in workers.items(): if w and hasattr(w, 'pause') and not getattr(w, 'paused', True): @@ -15499,7 +14355,7 @@ def _resume_workers_after_scan(): 'mb': mb_worker, 'spotify': spotify_enrichment_worker, 'itunes': itunes_enrichment_worker, 'deezer': deezer_worker, 'audiodb': audiodb_worker, 'discogs': discogs_worker, 'lastfm': lastfm_worker, 'genius': genius_worker, 'tidal': tidal_enrichment_worker, 'qobuz': qobuz_enrichment_worker, - 'repair': repair_worker, 'soulid': soulid_worker, + 'amazon': amazon_worker, 'repair': repair_worker, 'soulid': soulid_worker, } resumed = 0 for name, w in workers.items(): @@ -16948,8 +15804,14 @@ def musicbrainz_search_api(): mb_client = mb_svc.mb_client results = [] + # Manual Fix popup is user-facing fuzzy search — recall matters more + # than precision because the user picks the right hit from the list. + # Use bare-query mode so diacritics, aliases, and bracketed suffixes + # like "(Live)" don't kill matches the way strict field-scoped + # phrase queries do. Enrichment workers stay on strict mode (the + # default) since they auto-accept the top hit and need precision. if entity_type == 'artist': - raw = mb_client.search_artist(query, limit=limit) + raw = mb_client.search_artist(query, limit=limit, strict=False) for r in raw: results.append({ 'mbid': r.get('id', ''), @@ -16960,7 +15822,7 @@ def musicbrainz_search_api(): 'country': r.get('country', ''), }) elif entity_type == 'release': - raw = mb_client.search_release(query, artist_name=artist or None, limit=limit) + raw = mb_client.search_release(query, artist_name=artist or None, limit=limit, strict=False) for r in raw: artist_credit = ', '.join(a.get('name', '') for a in r.get('artist-credit', []) if isinstance(a, dict)) results.append({ @@ -16974,7 +15836,7 @@ def musicbrainz_search_api(): 'track_count': r.get('track-count', 0), }) elif entity_type == 'recording': - raw = mb_client.search_recording(query, artist_name=artist or None, limit=limit) + raw = mb_client.search_recording(query, artist_name=artist or None, limit=limit, strict=False) for r in raw: artist_credit = ', '.join(a.get('name', '') for a in r.get('artist-credit', []) if isinstance(a, dict)) releases = r.get('releases', []) @@ -16997,6 +15859,33 @@ def musicbrainz_search_api(): return jsonify({"error": str(e)}), 500 +# Recording MBID format: standard UUID, 8-4-4-4-12 hex. +_MB_RECORDING_MBID_RE = re.compile( + r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.IGNORECASE +) + + +@app.route('/api/musicbrainz/recording/', methods=['GET']) +def musicbrainz_recording_lookup_api(mbid): + """Look up a single MusicBrainz recording by MBID and return it in the + Fix-popup-compatible flat track shape. Powers the MBID-paste field — + user-facing escape hatch when fuzzy auto-search ranks the wrong + recording among many same-title versions.""" + mbid = (mbid or '').strip().lower() + if not _MB_RECORDING_MBID_RE.match(mbid): + return jsonify({"error": "Invalid MusicBrainz recording MBID"}), 400 + try: + from core.musicbrainz_search import MusicBrainzSearchClient + mb_search = MusicBrainzSearchClient() + track = mb_search.get_recording_flat(mbid) + if not track: + return jsonify({"error": "Recording not found on MusicBrainz"}), 404 + return jsonify(track) + except Exception as e: + logger.error(f"Error looking up MB recording {mbid}: {e}") + return jsonify({"error": str(e)}), 500 + + @app.route('/api/metadata-cache/mb-match', methods=['POST']) def metadata_cache_save_mb_match(): """Save a manual MusicBrainz match for a failed lookup.""" @@ -17610,9 +16499,10 @@ def _get_staging_file_cache(batch_id): """Scan staging folder once per batch and cache the results.""" with _staging_cache_lock: if batch_id in _staging_cache: - return _staging_cache[batch_id] + cached = _staging_cache[batch_id] + return [f for f in cached if os.path.exists(f.get('full_path', ''))] - staging_path = get_staging_path() + staging_path = _get_album_bundle_staging_path(batch_id) or get_staging_path() if not os.path.isdir(staging_path): with _staging_cache_lock: _staging_cache[batch_id] = [] @@ -17643,10 +16533,32 @@ def _get_staging_file_cache(batch_id): return files +def _get_album_bundle_staging_path(batch_id): + """Return the private album-bundle staging path for this batch, if any.""" + if not batch_id: + return None + with tasks_lock: + row = download_batches.get(batch_id) + if isinstance(row, dict) and row.get('album_bundle_private_staging'): + return row.get('album_bundle_staging_path') + return None + + # Staging-folder match shortcut lives in core/downloads/staging.py. from core.downloads import staging as _downloads_staging +def _staging_get_batch_field(batch_id, field): + """Accessor injected into StagingDeps so the staging-match helper + can read an album-bundle provenance override off the batch state + without importing ``download_batches`` directly.""" + with tasks_lock: + row = download_batches.get(batch_id) + if isinstance(row, dict): + return row.get(field) + return None + + def _build_staging_deps(): """Build the StagingDeps bundle from web_server.py globals on each call.""" return _downloads_staging.StagingDeps( @@ -17655,6 +16567,7 @@ def _build_staging_deps(): get_staging_file_cache=_get_staging_file_cache, docker_resolve_path=docker_resolve_path, post_process_matched_download_with_verification=_post_process_matched_download_with_verification, + get_batch_field=_staging_get_batch_field, ) @@ -17692,7 +16605,7 @@ def _try_source_reuse(task_id, batch_id, track): if not source_tracks or not last_source: _sr.info("Skipped — no source_tracks or no last_source") return False - if last_source.get('username') in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): + if last_source.get('username') in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): _sr.info(f"Skipped — {last_source.get('username')} source (no folder-based reuse)") return False @@ -17794,7 +16707,7 @@ def _store_batch_source(batch_id, username, filename): """Browse the successful download's folder and store results on the batch for reuse.""" _sr = source_reuse_logger _sr.info(f"_store_batch_source called: batch={batch_id}, user={username}, file={filename}") - if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): + if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', 'torrent', 'usenet'): _sr.info(f"Skipped — no batch_id or streaming source ({username})") return @@ -18161,6 +17074,10 @@ def cancel_download_task(): # Optionally try to cancel the Soulseek download (don't block worker progression) if download_id and username: try: + logger.info( + f"[CancelTrigger:web.cancel_download_task] " + f"download_id={download_id} username={username} task_id={task_id}" + ) # This is an async call, so we run it and wait run_async(download_orchestrator.cancel_download(download_id, username, remove=True)) logger.warning(f"Successfully cancelled Soulseek download {download_id} for task {task_id}") @@ -18413,6 +17330,10 @@ def cancel_task_v2(): # and silently left streaming downloads running in background. try: logger.info(f"[Atomic Cancel] Dispatching cancel to orchestrator: username={username} download_id={download_id}") + logger.info( + f"[CancelTrigger:web.atomic_cancel_v2] " + f"download_id={download_id} username={username}" + ) cancel_success = run_async( download_orchestrator.cancel_download(download_id, username, remove=True) ) @@ -18900,6 +17821,10 @@ def get_server_playlist_tracks(playlist_id): 'image_url': img, 'duration_ms': t.get('duration_ms', 0), 'position': t.get('position', 0), + # Spotify track id — required for the user-confirmed + # match override lookup (sync_match_cache). Null for + # iTunes-only sources. + 'source_track_id': t.get('source_track_id') or '', }) elif playlist_name: # Legacy fallback: cross-reference with sync history @@ -18939,6 +17864,21 @@ def get_server_playlist_tracks(playlist_id): used_server_indices = set() unmatched_source = [] # (index_in_combined, src_dict) for fuzzy second pass + # Pass 0: User-confirmed match overrides from sync_match_cache. + # When a user previously picked a local file via "Find & Add", + # the (source_track_id → server_track_id) mapping was persisted + # at confidence=1.0. Apply those FIRST so they bypass the + # exact/fuzzy passes entirely. Stale-cache safe — if the cached + # server track no longer exists, the override is silently + # skipped and normal matching runs. + from core.sync.match_overrides import resolve_match_overrides + _db_for_overrides = get_database() + _override_pairs = resolve_match_overrides( + source_tracks, + server_tracks, + lambda src_id: ((_db_for_overrides.read_sync_match_cache(src_id, active_server) or {}).get('server_track_id')), + ) + # Pass 1: Exact title match (normalized — strips feat./ft. qualifiers) for i, src in enumerate(source_tracks): src_name = src.get('name', '') @@ -18953,6 +17893,19 @@ def get_server_playlist_tracks(playlist_id): 'duration_ms': src.get('duration_ms', 0), 'position': src.get('position', i), } + # Override hit — paired by user, skip exact/fuzzy matching. + if i in _override_pairs: + j_override = _override_pairs[i] + used_server_indices.add(j_override) + combined.append({ + 'source_track': src_entry, + 'server_track': server_tracks[j_override], + 'match_status': 'matched', + 'confidence': 1.0, + 'override': True, + }) + continue + src_norm = _norm_title(src_name) best_idx = -1 for j, svr in enumerate(server_tracks): @@ -19125,14 +18078,48 @@ def server_playlist_replace_track(playlist_id): return jsonify({"success": False, "error": str(e)}), 500 +def _persist_find_and_add_match(source_track_id, server_source, server_track_id, server_track_title, source_title, source_artist): + """Wrap match-override persistence with the active DB. No-op when + source_track_id is missing (e.g. add to a non-mirrored playlist).""" + if not source_track_id: + return + try: + from core.sync.match_overrides import record_manual_match + ok = record_manual_match( + get_database(), + source_track_id=source_track_id, + server_source=server_source, + server_track_id=server_track_id, + server_track_title=server_track_title, + source_title=source_title, + source_artist=source_artist, + ) + if ok: + logger.info(f"[ServerPlaylist] Persisted Find & Add override: {source_track_id} → {server_track_id} ({server_source})") + except Exception as e: + logger.warning(f"[ServerPlaylist] Failed to persist Find & Add override: {e}") + + @app.route('/api/server/playlist//add-track', methods=['POST']) def server_playlist_add_track(playlist_id): - """Add a track to a server playlist at a specific position.""" + """Add a track to a server playlist at a specific position. + + When the optional `source_track_id` is provided (the Spotify track id + from a mirrored playlist), the user's selection is also persisted to + sync_match_cache so future syncs auto-match this source→server pair + without requiring the user to re-trigger Find & Add. + """ try: data = request.get_json() track_id = data.get('track_id') playlist_name = data.get('playlist_name', '') position = data.get('position') # 0-based index; None = append + # Optional Spotify source track id — when present, the (source → + # server) mapping is persisted as a hard match override. + source_track_id = data.get('source_track_id') or '' + source_title = data.get('source_title') or '' + source_artist = data.get('source_artist') or '' + server_track_title = data.get('server_track_title') or '' if not track_id: return jsonify({"success": False, "error": "track_id required"}), 400 @@ -19185,6 +18172,7 @@ def server_playlist_add_track(playlist_id): new_id = str(raw_playlist.ratingKey) logger.info(f"[ServerPlaylist] Added track to playlist, playlist ID: {new_id}") + _persist_find_and_add_match(source_track_id, active_server, track_id, server_track_title or new_item.title, source_title, source_artist) return jsonify({"success": True, "message": "Track added", "new_playlist_id": new_id}) elif active_server == 'jellyfin' and media_server_engine.client('jellyfin'): @@ -19194,6 +18182,7 @@ def server_playlist_add_track(playlist_id): track_ids.insert(pos, track_id) new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in track_ids] media_server_engine.client('jellyfin').update_playlist(playlist_name, new_track_objs) + _persist_find_and_add_match(source_track_id, active_server, track_id, server_track_title, source_title, source_artist) return jsonify({"success": True, "message": "Track added"}) elif active_server == 'navidrome' and media_server_engine.client('navidrome'): @@ -19203,6 +18192,7 @@ def server_playlist_add_track(playlist_id): track_ids.insert(pos, track_id) new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in track_ids] media_server_engine.client('navidrome').create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) + _persist_find_and_add_match(source_track_id, active_server, track_id, server_track_title, source_title, source_artist) return jsonify({"success": True, "message": "Track added"}) return jsonify({"success": False, "error": f"Unsupported server: {active_server}"}), 400 @@ -19336,6 +18326,100 @@ def library_search_tracks(): return jsonify({"success": False, "error": str(e)}), 500 +@app.route('/api/manual-library-matches', methods=['GET']) +def mlm_list(): + """List manual library matches for the current profile.""" + try: + from core.library import manual_library_match as mlm + db = get_database() + profile_id = get_current_profile_id() + return jsonify({"success": True, "matches": mlm.list_matches(db, profile_id)}) + except Exception as e: + logger.error(f"mlm_list error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/manual-library-matches/source-search', methods=['GET']) +def mlm_source_search(): + """Search wishlist + sync history for source track candidates.""" + try: + from core.library import manual_library_match as mlm + q = request.args.get('q', '').strip() + limit = min(int(request.args.get('limit', 15)), 50) + db = get_database() + profile_id = get_current_profile_id() + return jsonify({"success": True, "tracks": mlm.search_source_candidates(db, q, profile_id, limit)}) + except Exception as e: + logger.error(f"mlm_source_search error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/manual-library-matches/library-search', methods=['GET']) +def mlm_library_search(): + """Search the local library for candidate tracks.""" + try: + from core.library import manual_library_match as mlm + q = request.args.get('q', '').strip() + limit = min(int(request.args.get('limit', 15)), 50) + db = get_database() + return jsonify({"success": True, "tracks": mlm.search_library_candidates(db, q, limit)}) + except Exception as e: + logger.error(f"mlm_library_search error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/manual-library-matches', methods=['POST']) +def mlm_save(): + """Create or replace a manual library match.""" + try: + from core.library import manual_library_match as mlm + data = request.get_json(force=True) or {} + source = data.get('source', '').strip() + source_track_id = data.get('source_track_id', '').strip() + library_track_id = data.get('library_track_id') + if not source or not source_track_id or not library_track_id: + return jsonify({"success": False, "error": "source, source_track_id, library_track_id required"}), 400 + try: + library_track_id = int(library_track_id) + except (TypeError, ValueError): + return jsonify({"success": False, "error": "Invalid library track id"}), 400 + db = get_database() + profile_id = get_current_profile_id() + # Validate library track exists before saving + if not db.api_get_tracks_by_ids([library_track_id]): + return jsonify({"success": False, "error": "Library track not found — it may have been removed"}), 400 + ok = mlm.save_match( + db, profile_id, source, source_track_id, library_track_id, + source_title=data.get('source_title', ''), + source_artist=data.get('source_artist', ''), + source_album=data.get('source_album', ''), + source_context_json=data.get('source_context_json', ''), + server_source=data.get('server_source', ''), + ) + if not ok: + return jsonify({"success": False, "error": "Failed to save match"}), 500 + match = db.get_manual_library_match(profile_id, source, source_track_id, data.get('server_source', '')) + enriched = mlm._enrich_match(match, db) if match else {} + return jsonify({"success": True, "match": enriched}) + except Exception as e: + logger.error(f"mlm_save error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/manual-library-matches/', methods=['DELETE']) +def mlm_delete(match_id): + """Delete a manual library match by ID.""" + try: + from core.library import manual_library_match as mlm + db = get_database() + profile_id = get_current_profile_id() + mlm.delete_match(db, match_id, profile_id) + return jsonify({"success": True}) + except Exception as e: + logger.error(f"mlm_delete error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + @app.route('/api/playlists//start-missing-process', methods=['POST']) def start_missing_tracks_process(playlist_id): """ @@ -19348,6 +18432,9 @@ def start_missing_tracks_process(playlist_id): force_download_all = data.get('force_download_all', False) playlist_folder_mode = data.get('playlist_folder_mode', False) wing_it = data.get('wing_it', False) + ignore_manual_matches = data.get('ignore_manual_matches') + if ignore_manual_matches is None: + ignore_manual_matches = bool(force_download_all and not wing_it) # Get album/artist context for artist album downloads is_album_download = data.get('is_album_download', False) @@ -19396,12 +18483,14 @@ def start_missing_tracks_process(playlist_id): 'analysis_processed': 0, 'analysis_results': [], 'force_download_all': force_download_all, # Pass the force flag to the batch + 'ignore_manual_matches': ignore_manual_matches, 'playlist_folder_mode': playlist_folder_mode, # Organize downloads by playlist folder # Album context for artist album downloads (explicit folder structure) 'is_album_download': is_album_download, 'album_context': album_context, 'artist_context': artist_context, 'wing_it': wing_it, + 'batch_source': _downloads_history.detect_sync_source(playlist_id), } # Record sync history — derive source_page from context @@ -19875,6 +18964,9 @@ def get_spotify_album_tracks(album_id): client = _get_deezer_client() elif source_override == 'discogs': client = _get_discogs_client() + elif source_override == 'amazon': + from core.metadata.registry import get_amazon_client + client = get_amazon_client() elif source_override == 'musicbrainz': try: from core.musicbrainz_search import MusicBrainzSearchClient @@ -19891,8 +18983,12 @@ def get_spotify_album_tracks(album_id): if not album_data: return jsonify({"error": "Album not found"}), 404 - # Extract tracks from album data (Spotify format) - tracks = album_data.get('tracks', {}).get('items', []) + # Extract tracks — handle Spotify {items, total} or flat-list formats + tracks_container = album_data.get('tracks', {}) + if isinstance(tracks_container, list): + tracks = tracks_container + else: + tracks = tracks_container.get('items', []) # If no tracks in album data (iTunes format), fetch them separately if not tracks: @@ -20196,6 +19292,69 @@ def search_deezer_tracks(): return jsonify({"error": str(e)}), 500 +@app.route('/api/musicbrainz/search_tracks', methods=['GET']) +def search_musicbrainz_tracks(): + """Search for tracks on MusicBrainz — used by the Discovery Fix popup + cascade and any future surface that needs track-level MB search in the + Fix-popup track shape. + + Mirrors the spotify / itunes / deezer search_tracks endpoints exactly: + accepts `track` + `artist` (or legacy `query`) plus `limit`, returns + `{tracks: [{id, name, artists, album, duration_ms, image_url, source}]}`. + + Uses MB's bare-query mode for max recall (diacritic-folded, + alias/sortname indexed) — same rationale as the manual MBID-paste + endpoint shipped earlier. The Fix popup is a user-facing fuzzy search + where the user picks from the result list, so recall beats precision. + """ + try: + track_q = request.args.get('track', '').strip() + artist_q = request.args.get('artist', '').strip() + legacy_query = request.args.get('query', '').strip() + limit = int(request.args.get('limit', 20)) + + if not (track_q or artist_q or legacy_query): + return jsonify({"error": "Query parameter is required"}), 400 + + from core.musicbrainz_search import MusicBrainzSearchClient + mb_search = MusicBrainzSearchClient() + + if track_q or artist_q: + tracks = mb_search.search_tracks_with_artist( + track_q or legacy_query, artist_q, limit=limit + ) + else: + # Legacy single-string query — let MB's structured-query + # dispatch decide artist-first browse vs text search. + tracks = mb_search.search_tracks(legacy_query, limit=limit) + + # Local rerank — same helper Deezer / iTunes use. Penalises + # cover / karaoke / tribute patterns + boosts exact-artist match. + if track_q or artist_q: + from core.metadata.relevance import rerank_tracks + tracks = rerank_tracks( + tracks, + expected_title=track_q, + expected_artist=artist_q, + ) + + tracks_dict = [{ + 'id': t.id, + 'name': t.name, + 'artists': t.artists, + 'album': t.album, + 'duration_ms': t.duration_ms, + 'image_url': t.image_url, + 'source': 'musicbrainz', + } for t in tracks] + + return jsonify({'tracks': tracks_dict}) + + except Exception as e: + logger.error(f"Error searching MusicBrainz tracks: {e}") + return jsonify({"error": str(e)}), 500 + + @app.route('/api/itunes/album/', methods=['GET']) def get_itunes_album_tracks(album_id): """Fetches full track details for a specific iTunes album.""" @@ -20624,39 +19783,12 @@ def soundcloud_status(): @app.route('/api/hifi/instances', methods=['GET']) def hifi_instances(): """Check availability of all HiFi API instances.""" - import requests as req try: hifi = download_orchestrator.client("hifi") instances = list(hifi._instances) results = [] for url in instances: - entry = {'url': url, 'status': 'unknown', 'version': None, 'can_search': False, 'can_download': False} - try: - # Check root for version - r = req.get(f'{url}/', timeout=5, headers={'Accept': 'application/json'}) - if r.ok: - data = r.json() - entry['version'] = data.get('version') - entry['status'] = 'online' - # Check search - sr = req.get(f'{url}/search', params={'s': 'test', 'limit': 1}, timeout=5) - entry['can_search'] = sr.ok - # Check track (download capability) - tr = req.get(f'{url}/track', params={'id': '1550546', 'quality': 'LOSSLESS'}, timeout=5) - entry['can_download'] = tr.ok - if not tr.ok: - entry['download_error'] = f'HTTP {tr.status_code}' - else: - entry['status'] = f'error (HTTP {r.status_code})' - except req.exceptions.SSLError: - entry['status'] = 'ssl_error' - except req.exceptions.ConnectTimeout: - entry['status'] = 'timeout' - except req.exceptions.ConnectionError: - entry['status'] = 'offline' - except Exception as e: - entry['status'] = f'error ({type(e).__name__})' - results.append(entry) + results.append(hifi.check_instance_capabilities(url)) return jsonify({'instances': results, 'active': hifi._get_instance()}) except Exception as e: return jsonify({'error': str(e)}), 500 @@ -20895,6 +20027,26 @@ def deezer_download_test_download(): # =================================================================== +# AMAZON DOWNLOAD ENDPOINTS +# =================================================================== + +@app.route('/api/amazon/test-connection', methods=['GET']) +@admin_only +def amazon_test_connection(): + """Check whether the T2Tunes proxy is up and Amazon Music is reachable.""" + try: + from core.amazon_client import AmazonClient + c = AmazonClient() + status = c.status() + amazon_up = str(status.get('amazonMusic', '')).lower() == 'up' + return jsonify({ + 'connected': amazon_up, + 'status': status, + }) + except Exception as e: + return jsonify({'connected': False, 'error': str(e)}), 200 + + # TIDAL DOWNLOAD AUTH ENDPOINTS # =================================================================== @@ -22696,6 +21848,655 @@ def cancel_deezer_sync(playlist_id): return jsonify({"error": str(e)}), 500 +# =================================================================== +# QOBUZ PLAYLIST DISCOVERY API ENDPOINTS +# =================================================================== +# +# Mirrors the Tidal + Deezer endpoint set for parity on the Sync page. +# Qobuz playlists arrive from `core/qobuz_client.py` as dicts (matching +# the Deezer client's shape), so the state + endpoint code follows the +# Deezer template rather than Tidal's dataclass-based one. Github issue +# #677. + +# Global state for Qobuz playlist discovery management +qobuz_discovery_states = {} # Key: playlist_id, Value: discovery state +qobuz_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="qobuz_discovery") + + +def _get_qobuz_client_for_sync(): + """Resolve the Qobuz client via the download orchestrator. + + The orchestrator owns the canonical instance (same one Settings → + Connections authenticates against), so the Sync page tab always sees + fresh auth state without a second login flow. + """ + if not download_orchestrator or not hasattr(download_orchestrator, 'client'): + return None + try: + return download_orchestrator.client("qobuz") + except Exception as exc: + logger.debug(f"Qobuz client lookup failed: {exc}") + return None + + +@app.route('/api/qobuz/playlists', methods=['GET']) +def get_qobuz_playlists(): + """Fetches the authenticated user's Qobuz playlists (metadata only). + + Tracks are fetched on demand by the per-playlist detail endpoint — + matches the Tidal + Deezer behaviour so the Sync page renderer can + treat all three services uniformly. + """ + qobuz = _get_qobuz_client_for_sync() + if not qobuz or not qobuz.is_authenticated(): + return jsonify({"error": "Qobuz not authenticated."}), 401 + + try: + playlists = qobuz.get_user_playlists() + + playlist_data = [] + for p in playlists: + playlist_data.append({ + "id": p['id'], + "name": p['name'], + "owner": "You", + "track_count": p.get('track_count', 0), + "image_url": p.get('image_url') or None, + "description": p.get('description', ''), + "tracks": [] + }) + + # Append virtual "Favorite Tracks" entry at the END (mirrors + # Tidal's COLLECTION_PLAYLIST_ID pattern — count only here, full + # fetch deferred to the per-playlist detail endpoint). + try: + from core.qobuz_client import QobuzClient as _QobuzClientTypeRef + favorites_count = qobuz.get_user_favorite_tracks_count() + if favorites_count > 0: + playlist_data.append({ + "id": qobuz.QOBUZ_FAVORITES_ID, + "name": qobuz.QOBUZ_FAVORITES_NAME, + "owner": "You", + "track_count": favorites_count, + "image_url": None, + "description": qobuz.QOBUZ_FAVORITES_DESCRIPTION, + "tracks": [], + }) + logger.info( + f"Added virtual '{qobuz.QOBUZ_FAVORITES_NAME}' playlist with {favorites_count} tracks (count only)" + ) + except Exception as favorites_error: + logger.error(f"Failed to add Qobuz Favorite Tracks playlist: {favorites_error}") + + logger.info(f"Loaded {len(playlist_data)} Qobuz playlists") + return jsonify(playlist_data) + except Exception as e: + logger.error(f"Error loading Qobuz playlists: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/playlist/', methods=['GET']) +def get_qobuz_playlist_tracks(playlist_id): + """Fetches full track details for a specific Qobuz playlist.""" + qobuz = _get_qobuz_client_for_sync() + if not qobuz or not qobuz.is_authenticated(): + return jsonify({"error": "Qobuz not authenticated."}), 401 + + try: + logger.info(f"Getting full Qobuz playlist with tracks for: {playlist_id}") + full_playlist = qobuz.get_playlist(playlist_id) + if not full_playlist: + return jsonify({"error": "Playlist not found or unable to access."}), 404 + + tracks = full_playlist.get('tracks') or [] + if not tracks: + return jsonify({"error": "This playlist appears to have no tracks or they cannot be accessed"}), 403 + + logger.info(f"Loaded {len(tracks)} tracks from Qobuz playlist: {full_playlist['name']}") + + playlist_dict = { + 'id': full_playlist['id'], + 'name': full_playlist['name'], + 'description': full_playlist.get('description', ''), + 'owner': 'You', + 'track_count': len(tracks), + 'image_url': full_playlist.get('image_url') or None, + 'tracks': tracks, + } + return jsonify(playlist_dict) + except Exception as e: + logger.error(f"Error getting Qobuz playlist tracks: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/discovery/start/', methods=['POST']) +def start_qobuz_discovery(playlist_id): + """Start Spotify discovery process for a Qobuz playlist.""" + try: + qobuz = _get_qobuz_client_for_sync() + if not qobuz or not qobuz.is_authenticated(): + return jsonify({"error": "Qobuz not authenticated."}), 401 + + if playlist_id in qobuz_discovery_states: + existing_state = qobuz_discovery_states[playlist_id] + if existing_state['phase'] == 'discovering': + return jsonify({"error": "Discovery already in progress"}), 400 + + if not existing_state.get('playlist'): + playlist_data = qobuz.get_playlist(playlist_id) + if not playlist_data: + return jsonify({"error": "Qobuz playlist not found"}), 404 + existing_state['playlist'] = playlist_data + + existing_state['phase'] = 'discovering' + existing_state['status'] = 'discovering' + existing_state['last_accessed'] = time.time() + state = existing_state + else: + playlist_data = qobuz.get_playlist(playlist_id) + + if not playlist_data: + return jsonify({"error": "Qobuz playlist not found"}), 404 + + if not playlist_data.get('tracks'): + return jsonify({"error": "Playlist has no tracks"}), 400 + + state = { + 'playlist': playlist_data, + 'phase': 'discovering', + 'status': 'discovering', + 'discovery_progress': 0, + 'spotify_matches': 0, + 'spotify_total': len(playlist_data['tracks']), + 'discovery_results': [], + 'sync_playlist_id': None, + 'converted_spotify_playlist_id': None, + 'download_process_id': None, + 'created_at': time.time(), + 'last_accessed': time.time(), + 'discovery_future': None, + 'sync_progress': {} + } + qobuz_discovery_states[playlist_id] = state + + playlist_name = state['playlist']['name'] + track_count = len(state['playlist']['tracks']) + add_activity_item("", "Qobuz Discovery Started", f"'{playlist_name}' - {track_count} tracks", "Now") + + qobuz_discovery_states[playlist_id]['_profile_id'] = get_current_profile_id() + future = qobuz_discovery_executor.submit(_run_qobuz_discovery_worker, playlist_id) + state['discovery_future'] = future + + logger.info(f"Started Spotify discovery for Qobuz playlist: {playlist_name}") + return jsonify({"success": True, "message": "Discovery started"}) + + except Exception as e: + logger.error(f"Error starting Qobuz discovery: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/discovery/status/', methods=['GET']) +def get_qobuz_discovery_status(playlist_id): + """Get real-time discovery status for a Qobuz playlist.""" + try: + if playlist_id not in qobuz_discovery_states: + return jsonify({"error": "Qobuz discovery not found"}), 404 + + state = qobuz_discovery_states[playlist_id] + state['last_accessed'] = time.time() + + response = { + 'phase': state['phase'], + 'status': state['status'], + 'progress': state['discovery_progress'], + 'spotify_matches': state['spotify_matches'], + 'spotify_total': state['spotify_total'], + 'results': state['discovery_results'], + 'complete': state['phase'] == 'discovered' + } + + return jsonify(response) + + except Exception as e: + logger.error(f"Error getting Qobuz discovery status: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/discovery/update_match', methods=['POST']) +def update_qobuz_discovery_match(): + """Update a Qobuz discovery result with a manually selected Spotify track.""" + try: + data = request.get_json() + identifier = data.get('identifier') # playlist_id + track_index = data.get('track_index') + spotify_track = data.get('spotify_track') + + if not identifier or track_index is None or not spotify_track: + return jsonify({'error': 'Missing required fields'}), 400 + + state = qobuz_discovery_states.get(identifier) + if not state: + return jsonify({'error': 'Discovery state not found'}), 404 + + if track_index >= len(state['discovery_results']): + return jsonify({'error': 'Invalid track index'}), 400 + + result = state['discovery_results'][track_index] + old_status = result.get('status') + + result['status'] = 'Found' + result['status_class'] = 'found' + result['spotify_track'] = spotify_track['name'] + result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) + result['spotify_album'] = spotify_track['album'] + result['spotify_id'] = spotify_track['id'] + + duration_ms = spotify_track.get('duration_ms', 0) + if duration_ms: + minutes = duration_ms // 60000 + seconds = (duration_ms % 60000) // 1000 + result['duration'] = f"{minutes}:{seconds:02d}" + else: + result['duration'] = '0:00' + + # Manual match from the fix modal — build rich spotify_data matching + # the normal discovery shape, clear wing-it flag since the user + # picked a real metadata match. + result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) + result['wing_it_fallback'] = False + result['manual_match'] = True + + if old_status != 'found' and old_status != 'Found': + state['spotify_matches'] = state.get('spotify_matches', 0) + 1 + + logger.info(f"Manual match updated: qobuz - {identifier} - track {track_index}") + logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") + + try: + original_track = result.get('qobuz_track', {}) + original_name = original_track.get('name', spotify_track['name']) + original_artists = original_track.get('artists', []) + original_artist = original_artists[0] if original_artists else '' + + cache_key = _get_discovery_cache_key(original_name, original_artist) + artists_list = spotify_track['artists'] + if isinstance(artists_list, list): + artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] + image_url = spotify_track.get('image_url') or '' + album_raw = spotify_track.get('album', '') + if isinstance(album_raw, dict): + album_obj = dict(album_raw) + if image_url and not album_obj.get('image_url'): + album_obj['image_url'] = image_url + if image_url and not album_obj.get('images'): + album_obj['images'] = [{'url': image_url}] + else: + album_obj = {'name': album_raw or ''} + if image_url: + album_obj['image_url'] = image_url + album_obj['images'] = [{'url': image_url}] + + matched_data = { + 'id': spotify_track['id'], + 'name': spotify_track['name'], + 'artists': artists_list, + 'album': album_obj, + 'duration_ms': spotify_track.get('duration_ms', 0), + 'image_url': image_url, + 'source': 'spotify', + } + cache_db = get_database() + cache_db.save_discovery_cache_match( + cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, + original_name, original_artist + ) + logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") + except Exception as cache_err: + logger.error(f"Error saving manual fix to discovery cache: {cache_err}") + + return jsonify({'success': True, 'result': result}) + + except Exception as e: + logger.error(f"Error updating Qobuz discovery match: {e}") + return jsonify({'error': str(e)}), 500 + + +@app.route('/api/qobuz/playlists/states', methods=['GET']) +def get_qobuz_playlist_states(): + """Get all stored Qobuz playlist discovery states for frontend hydration.""" + try: + states = [] + current_time = time.time() + + for playlist_id, state in qobuz_discovery_states.items(): + state['last_accessed'] = current_time + + state_info = { + 'playlist_id': playlist_id, + 'phase': state['phase'], + 'status': state['status'], + 'discovery_progress': state['discovery_progress'], + 'spotify_matches': state['spotify_matches'], + 'spotify_total': state['spotify_total'], + 'discovery_results': state['discovery_results'], + 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), + 'download_process_id': state.get('download_process_id'), + 'last_accessed': state['last_accessed'] + } + states.append(state_info) + + logger.info(f"Returning {len(states)} stored Qobuz playlist states for hydration") + return jsonify({"states": states}) + + except Exception as e: + logger.error(f"Error getting Qobuz playlist states: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/state/', methods=['GET']) +def get_qobuz_playlist_state(playlist_id): + """Get specific Qobuz playlist state (detailed version).""" + try: + if playlist_id not in qobuz_discovery_states: + return jsonify({"error": "Qobuz playlist not found"}), 404 + + state = qobuz_discovery_states[playlist_id] + state['last_accessed'] = time.time() + + response = { + 'playlist_id': playlist_id, + 'playlist': state['playlist'], + 'phase': state['phase'], + 'status': state['status'], + 'discovery_progress': state['discovery_progress'], + 'spotify_matches': state['spotify_matches'], + 'spotify_total': state['spotify_total'], + 'discovery_results': state['discovery_results'], + 'sync_playlist_id': state.get('sync_playlist_id'), + 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), + 'download_process_id': state.get('download_process_id'), + 'sync_progress': state.get('sync_progress', {}), + 'last_accessed': state['last_accessed'] + } + + return jsonify(response) + + except Exception as e: + logger.error(f"Error getting Qobuz playlist state: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/reset/', methods=['POST']) +def reset_qobuz_playlist(playlist_id): + """Reset Qobuz playlist to fresh phase (clear discovery/sync data).""" + try: + if playlist_id not in qobuz_discovery_states: + return jsonify({"error": "Qobuz playlist not found"}), 404 + + state = qobuz_discovery_states[playlist_id] + + if 'discovery_future' in state and state['discovery_future']: + state['discovery_future'].cancel() + + state['phase'] = 'fresh' + state['status'] = 'fresh' + state['discovery_results'] = [] + state['discovery_progress'] = 0 + state['spotify_matches'] = 0 + state['sync_playlist_id'] = None + state['converted_spotify_playlist_id'] = None + state['download_process_id'] = None + state['sync_progress'] = {} + state['discovery_future'] = None + state['last_accessed'] = time.time() + + logger.info(f"Reset Qobuz playlist to fresh: {playlist_id}") + return jsonify({"success": True, "message": "Playlist reset to fresh phase"}) + + except Exception as e: + logger.error(f"Error resetting Qobuz playlist: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/delete/', methods=['POST']) +def delete_qobuz_playlist(playlist_id): + """Delete Qobuz playlist state completely.""" + try: + if playlist_id not in qobuz_discovery_states: + return jsonify({"error": "Qobuz playlist not found"}), 404 + + state = qobuz_discovery_states[playlist_id] + + if 'discovery_future' in state and state['discovery_future']: + state['discovery_future'].cancel() + + del qobuz_discovery_states[playlist_id] + + logger.info(f"Deleted Qobuz playlist state: {playlist_id}") + return jsonify({"success": True, "message": "Playlist deleted"}) + + except Exception as e: + logger.error(f"Error deleting Qobuz playlist: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/update_phase/', methods=['POST']) +def update_qobuz_playlist_phase(playlist_id): + """Update Qobuz playlist phase (used when modal closes to reset from download_complete to discovered).""" + try: + if playlist_id not in qobuz_discovery_states: + return jsonify({"error": "Qobuz playlist not found"}), 404 + + data = request.get_json() + if not data or 'phase' not in data: + return jsonify({"error": "Phase not provided"}), 400 + + new_phase = data['phase'] + valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] + + if new_phase not in valid_phases: + return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 + + state = qobuz_discovery_states[playlist_id] + old_phase = state.get('phase', 'unknown') + state['phase'] = new_phase + state['last_accessed'] = time.time() + + if 'download_process_id' in data: + state['download_process_id'] = data['download_process_id'] + if 'converted_spotify_playlist_id' in data: + state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id'] + + logger.info(f"Updated Qobuz playlist {playlist_id} phase: {old_phase} → {new_phase}") + return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) + + except Exception as e: + logger.error(f"Error updating Qobuz playlist phase: {e}") + return jsonify({"error": str(e)}), 500 + + +# Qobuz discovery worker logic lives in core/discovery/qobuz.py. +from core.discovery import qobuz as _discovery_qobuz + + +def _build_qobuz_discovery_deps(): + """Build the QobuzDiscoveryDeps bundle from web_server.py globals on each call.""" + return _discovery_qobuz.QobuzDiscoveryDeps( + qobuz_discovery_states=qobuz_discovery_states, + spotify_client=spotify_client, + pause_enrichment_workers=_pause_enrichment_workers, + resume_enrichment_workers=_resume_enrichment_workers, + get_active_discovery_source=_get_active_discovery_source, + get_metadata_fallback_client=_get_metadata_fallback_client, + get_discovery_cache_key=_get_discovery_cache_key, + get_database=get_database, + validate_discovery_cache_artist=_validate_discovery_cache_artist, + search_spotify_for_tidal_track=_search_spotify_for_tidal_track, + build_discovery_wing_it_stub=_build_discovery_wing_it_stub, + add_activity_item=add_activity_item, + sync_discovery_results_to_mirrored=_sync_discovery_results_to_mirrored, + ) + + +def _run_qobuz_discovery_worker(playlist_id): + return _discovery_qobuz.run_qobuz_discovery_worker(playlist_id, _build_qobuz_discovery_deps()) + + +def convert_qobuz_results_to_spotify_tracks(discovery_results): + """Convert Qobuz discovery results to Spotify tracks format for sync.""" + spotify_tracks = [] + + for result in discovery_results: + if result.get('spotify_data'): + spotify_data = result['spotify_data'] + + track = { + 'id': spotify_data['id'], + 'name': spotify_data['name'], + 'artists': spotify_data['artists'], + 'album': spotify_data['album'], + 'duration_ms': spotify_data.get('duration_ms', 0) + } + if spotify_data.get('track_number'): + track['track_number'] = spotify_data['track_number'] + if spotify_data.get('disc_number'): + track['disc_number'] = spotify_data['disc_number'] + spotify_tracks.append(track) + elif result.get('spotify_track') and result.get('status_class') == 'found': + track = { + 'id': result.get('spotify_id', 'unknown'), + 'name': result.get('spotify_track', 'Unknown Track'), + 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], + 'album': result.get('spotify_album', 'Unknown Album'), + 'duration_ms': 0 + } + spotify_tracks.append(track) + + logger.info(f"Converted {len(spotify_tracks)} Qobuz matches to Spotify tracks for sync") + return spotify_tracks + + +# =================================================================== +# QOBUZ SYNC API ENDPOINTS +# =================================================================== + +@app.route('/api/qobuz/sync/start/', methods=['POST']) +def start_qobuz_sync(playlist_id): + """Start sync process for a Qobuz playlist using discovered Spotify tracks.""" + try: + if playlist_id not in qobuz_discovery_states: + return jsonify({"error": "Qobuz playlist not found"}), 404 + + state = qobuz_discovery_states[playlist_id] + state['last_accessed'] = time.time() + + if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: + return jsonify({"error": "Qobuz playlist not ready for sync"}), 400 + + spotify_tracks = convert_qobuz_results_to_spotify_tracks(state['discovery_results']) + if not spotify_tracks: + return jsonify({"error": "No Spotify matches found for sync"}), 400 + + sync_playlist_id = f"qobuz_{playlist_id}" + playlist_name = state['playlist']['name'] + + add_activity_item("", "Qobuz Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") + + state['phase'] = 'syncing' + state['sync_playlist_id'] = sync_playlist_id + state['sync_progress'] = {} + + sync_data = { + 'playlist_id': sync_playlist_id, + 'playlist_name': playlist_name, + 'tracks': spotify_tracks + } + + with sync_lock: + sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} + + playlist_image_url = state['playlist'].get('image_url', '') + future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) + active_sync_workers[sync_playlist_id] = future + + logger.info(f"Started Qobuz sync for: {playlist_name} ({len(spotify_tracks)} tracks)") + return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) + + except Exception as e: + logger.error(f"Error starting Qobuz sync: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/sync/status/', methods=['GET']) +def get_qobuz_sync_status(playlist_id): + """Get sync status for a Qobuz playlist.""" + try: + if playlist_id not in qobuz_discovery_states: + return jsonify({"error": "Qobuz playlist not found"}), 404 + + state = qobuz_discovery_states[playlist_id] + state['last_accessed'] = time.time() + sync_playlist_id = state.get('sync_playlist_id') + + if not sync_playlist_id: + return jsonify({"error": "No sync in progress"}), 404 + + with sync_lock: + sync_state = sync_states.get(sync_playlist_id, {}) + + response = { + 'phase': state['phase'], + 'sync_status': sync_state.get('status', 'unknown'), + 'progress': sync_state.get('progress', {}), + 'complete': sync_state.get('status') == 'finished', + 'error': sync_state.get('error') + } + + if sync_state.get('status') == 'finished': + state['phase'] = 'sync_complete' + state['sync_progress'] = sync_state.get('progress', {}) + playlist_name = state['playlist']['name'] + add_activity_item("", "Sync Complete", f"Qobuz playlist '{playlist_name}' synced successfully", "Now") + elif sync_state.get('status') == 'error': + state['phase'] = 'discovered' + playlist_name = state['playlist']['name'] + add_activity_item("", "Sync Failed", f"Qobuz playlist '{playlist_name}' sync failed", "Now") + + return jsonify(response) + + except Exception as e: + logger.error(f"Error getting Qobuz sync status: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/qobuz/sync/cancel/', methods=['POST']) +def cancel_qobuz_sync(playlist_id): + """Cancel sync for a Qobuz playlist.""" + try: + if playlist_id not in qobuz_discovery_states: + return jsonify({"error": "Qobuz playlist not found"}), 404 + + state = qobuz_discovery_states[playlist_id] + state['last_accessed'] = time.time() + sync_playlist_id = state.get('sync_playlist_id') + + if sync_playlist_id: + with sync_lock: + sync_states[sync_playlist_id] = {"status": "cancelled"} + if sync_playlist_id in active_sync_workers: + del active_sync_workers[sync_playlist_id] + + state['phase'] = 'discovered' + state['sync_playlist_id'] = None + state['sync_progress'] = {} + + return jsonify({"success": True, "message": "Qobuz sync cancelled"}) + + except Exception as e: + logger.error(f"Error cancelling Qobuz sync: {e}") + return jsonify({"error": str(e)}), 500 + + # =================================================================== # SPOTIFY PUBLIC PLAYLIST DISCOVERY API ENDPOINTS # =================================================================== @@ -25167,9 +24968,10 @@ def select_profile(): return jsonify({'success': False, 'error': 'Invalid PIN'}), 401 session['profile_id'] = profile_id - # If PIN was just validated, also mark launch PIN as verified - # so the subsequent page reload doesn't ask again - if pin: + # If the admin PIN was just validated, also mark launch PIN as + # verified so the subsequent page reload doesn't ask again. A + # non-admin profile PIN must not unlock the admin launch lock. + if pin and profile_id == 1: session['launch_pin_verified'] = True return jsonify({'success': True, 'profile': profile}) except Exception as e: @@ -25254,12 +25056,16 @@ def reset_pin_via_credential(): # Credential verified — clear PIN for the requested profile (default: admin) database = get_database() - target_profile = data.get('profile_id', 1) + try: + target_profile = int(data.get('profile_id', 1)) + except (TypeError, ValueError): + target_profile = 1 database.update_profile(target_profile, pin_hash=None) # If clearing admin PIN, also disable launch lock if target_profile == 1: config_manager.set('security.require_pin_on_launch', False) - session['launch_pin_verified'] = True + if target_profile == 1: + session['launch_pin_verified'] = True return jsonify({'success': True, 'message': 'PIN cleared. You can set a new PIN in Settings.'}) except Exception as e: @@ -25531,6 +25337,7 @@ def get_watchlist_artists(): """Get all artists in the watchlist with cached images""" try: database = get_database() + database.backfill_watchlist_musicbrainz_ids_from_library(profile_id=get_current_profile_id()) watchlist_artists = database.get_watchlist_artists(profile_id=get_current_profile_id()) # Convert to JSON serializable format (images are cached from watchlist scans) @@ -25548,6 +25355,8 @@ def get_watchlist_artists(): "itunes_artist_id": artist.itunes_artist_id, # For iTunes-only artists "deezer_artist_id": getattr(artist, 'deezer_artist_id', None), "discogs_artist_id": getattr(artist, 'discogs_artist_id', None), + "musicbrainz_artist_id": getattr(artist, 'musicbrainz_artist_id', None), + "amazon_artist_id": getattr(artist, 'amazon_artist_id', None), "include_albums": artist.include_albums, "include_eps": artist.include_eps, "include_singles": artist.include_singles, @@ -25584,7 +25393,7 @@ def add_to_watchlist(): conn = database._get_connection() cursor = conn.cursor() cursor.execute(""" - SELECT spotify_artist_id, itunes_artist_id, deezer_id, discogs_id + SELECT spotify_artist_id, itunes_artist_id, deezer_id, discogs_id, musicbrainz_id FROM artists WHERE id = ? LIMIT 1 """, (artist_id,)) row = cursor.fetchone() @@ -25595,6 +25404,9 @@ def add_to_watchlist(): if fallback == 'discogs' and row['discogs_id']: artist_id = row['discogs_id'] source = 'discogs' + elif fallback == 'musicbrainz' and row['musicbrainz_id']: + artist_id = row['musicbrainz_id'] + source = 'musicbrainz' elif fallback == 'deezer' and row['deezer_id']: artist_id = row['deezer_id'] source = 'deezer' @@ -25610,12 +25422,17 @@ def add_to_watchlist(): elif row['discogs_id']: artist_id = row['discogs_id'] source = 'discogs' + elif row['musicbrainz_id']: + artist_id = row['musicbrainz_id'] + source = 'musicbrainz' except Exception as e: logger.debug("watchlist artist source lookup failed: %s", e) if not source: fallback_source = _get_metadata_fallback_source() source = fallback_source if is_numeric_id else 'spotify' success = database.add_artist_to_watchlist(artist_id, artist_name, profile_id=get_current_profile_id(), source=source) + if success: + database.backfill_watchlist_musicbrainz_ids_from_library(profile_id=get_current_profile_id()) if success: @@ -26033,7 +25850,7 @@ def start_watchlist_scan(): # PROACTIVE ID BACKFILLING (cross-provider support) # Before scanning, ensure all artists have IDs for ALL available sources - providers_to_backfill = ['itunes', 'deezer'] + providers_to_backfill = ['itunes', 'deezer', 'musicbrainz'] if spotify_client and spotify_client.is_spotify_authenticated(): providers_to_backfill.append('spotify') try: @@ -26339,6 +26156,7 @@ def watchlist_artist_config(artist_id): database = get_database() if request.method == 'GET': + database.backfill_watchlist_musicbrainz_ids_from_library(profile_id=get_current_profile_id()) # Get current config from database conn = sqlite3.connect(str(database.database_path)) cursor = conn.cursor() @@ -26347,10 +26165,12 @@ def watchlist_artist_config(artist_id): include_live, include_remixes, include_acoustic, include_compilations, artist_name, image_url, spotify_artist_id, itunes_artist_id, last_scan_timestamp, date_added, include_instrumentals, deezer_artist_id, - lookback_days, discogs_artist_id, preferred_metadata_source + lookback_days, discogs_artist_id, preferred_metadata_source, + amazon_artist_id, musicbrainz_artist_id FROM watchlist_artists - WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? - """, (artist_id, artist_id, artist_id, artist_id)) + WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR amazon_artist_id = ? OR musicbrainz_artist_id = ? + """, (artist_id, artist_id, artist_id, artist_id, artist_id, artist_id)) result = cursor.fetchone() conn.close() @@ -26359,10 +26179,12 @@ def watchlist_artist_config(artist_id): # Determine if this is an iTunes or Spotify artist is_itunes_artist = artist_id.isdigit() - spotify_id = result[9] # spotify_artist_id from query + spotify_id = result[9] # spotify_artist_id from query itunes_id = result[10] # itunes_artist_id from query deezer_id = result[14] # deezer_artist_id from query discogs_id = result[16] # discogs_artist_id from query + amazon_id = result[18] if len(result) > 18 else None # amazon_artist_id from query + musicbrainz_id = result[19] if len(result) > 19 else None # musicbrainz_artist_id from query # Get artist info from Spotify (only for Spotify artists) artist_info = None @@ -26405,9 +26227,16 @@ def watchlist_artist_config(artist_id): cur2.execute(""" SELECT banner_url, summary, style, mood, label, genres FROM artists - WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_id = ? OR discogs_id = ? + WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_id = ? + OR discogs_id = ? OR musicbrainz_id = ? LIMIT 1 - """, (artist_id, artist_id, artist_id, artist_id)) + """, ( + spotify_id or artist_id, + itunes_id or artist_id, + deezer_id or artist_id, + discogs_id or artist_id, + musicbrainz_id or artist_id, + )) lib_row = cur2.fetchone() if lib_row: artist_info['banner_url'] = lib_row[0] @@ -26428,9 +26257,17 @@ def watchlist_artist_config(artist_id): FROM recent_releases rr JOIN watchlist_artists wa ON rr.watchlist_artist_id = wa.id WHERE wa.spotify_artist_id = ? OR wa.itunes_artist_id = ? OR wa.deezer_artist_id = ? + OR wa.discogs_artist_id = ? OR wa.amazon_artist_id = ? OR wa.musicbrainz_artist_id = ? ORDER BY rr.release_date DESC LIMIT 6 - """, (artist_id, artist_id, artist_id)) + """, ( + spotify_id or artist_id, + itunes_id or artist_id, + deezer_id or artist_id, + discogs_id or artist_id, + amazon_id or artist_id, + musicbrainz_id or artist_id, + )) releases = [ { 'album_name': r[0], @@ -26470,6 +26307,8 @@ def watchlist_artist_config(artist_id): "itunes_artist_id": itunes_id, "deezer_artist_id": deezer_id, "discogs_artist_id": discogs_id, + "amazon_artist_id": amazon_id, + "musicbrainz_artist_id": musicbrainz_id, "watchlist_name": result[7], # Original stored watchlist artist name "global_metadata_source": get_primary_source(), }) @@ -26493,7 +26332,7 @@ def watchlist_artist_config(artist_id): lookback_days = int(lookback_days) if lookback_days != '' else None preferred_metadata_source = data.get('preferred_metadata_source', None) # Validate — only accept known sources, empty string means clear override - if preferred_metadata_source == '' or preferred_metadata_source not in ('spotify', 'deezer', 'itunes', 'discogs'): + if preferred_metadata_source == '' or preferred_metadata_source not in ('spotify', 'deezer', 'itunes', 'discogs', 'musicbrainz'): preferred_metadata_source = None # Validate at least one release type is selected @@ -26507,8 +26346,9 @@ def watchlist_artist_config(artist_id): # Check if lookback_days changed — if so, clear last_scan_timestamp to force rescan cursor.execute(""" SELECT lookback_days FROM watchlist_artists - WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? - """, (artist_id, artist_id, artist_id, artist_id)) + WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR musicbrainz_artist_id = ? + """, (artist_id, artist_id, artist_id, artist_id, artist_id)) old_row = cursor.fetchone() old_lookback = old_row[0] if old_row else None lookback_changed = old_lookback != lookback_days @@ -26520,11 +26360,12 @@ def watchlist_artist_config(artist_id): include_instrumentals = ?, lookback_days = ?, preferred_metadata_source = ?, last_scan_timestamp = CASE WHEN ? THEN NULL ELSE last_scan_timestamp END, updated_at = CURRENT_TIMESTAMP - WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? + WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR musicbrainz_artist_id = ? """, (int(include_albums), int(include_eps), int(include_singles), int(include_live), int(include_remixes), int(include_acoustic), int(include_compilations), int(include_instrumentals), lookback_days, preferred_metadata_source, lookback_changed, - artist_id, artist_id, artist_id, artist_id)) + artist_id, artist_id, artist_id, artist_id, artist_id)) conn.commit() if cursor.rowcount == 0: @@ -26570,7 +26411,7 @@ def watchlist_artist_link_provider(artist_id): new_provider_id = data.get('provider_id', '').strip() provider = data.get('provider', '').strip() - valid_providers = ('spotify', 'itunes', 'deezer', 'discogs') + valid_providers = ('spotify', 'itunes', 'deezer', 'discogs', 'amazon', 'musicbrainz') if provider not in valid_providers: return jsonify({"success": False, "error": f"Invalid provider. Must be one of: {', '.join(valid_providers)}"}), 400 @@ -26584,8 +26425,9 @@ def watchlist_artist_link_provider(artist_id): cursor.execute(""" SELECT id, artist_name, spotify_artist_id, itunes_artist_id FROM watchlist_artists - WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? - """, (artist_id, artist_id, artist_id, artist_id)) + WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? + OR discogs_artist_id = ? OR amazon_artist_id = ? OR musicbrainz_artist_id = ? + """, (artist_id, artist_id, artist_id, artist_id, artist_id, artist_id)) row = cursor.fetchone() if not row: @@ -26596,7 +26438,14 @@ def watchlist_artist_link_provider(artist_id): artist_name = row[1] # Check for duplicate — another watchlist artist already has this provider ID - col_map = {'spotify': 'spotify_artist_id', 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id'} + col_map = { + 'spotify': 'spotify_artist_id', + 'itunes': 'itunes_artist_id', + 'deezer': 'deezer_artist_id', + 'discogs': 'discogs_artist_id', + 'amazon': 'amazon_artist_id', + 'musicbrainz': 'musicbrainz_artist_id', + } col = col_map[provider] if not is_clear: @@ -26901,8 +26750,15 @@ def get_discover_similar_artists(): try: database = get_database() active_source = _get_active_discovery_source() + from config.settings import config_manager + active_server = config_manager.get_active_media_server() - similar_artists = database.get_top_similar_artists(limit=200, profile_id=get_current_profile_id(), require_source=active_source) + similar_artists = database.get_top_similar_artists( + limit=200, + profile_id=get_current_profile_id(), + require_source=active_source, + exclude_library_server=active_server, + ) if not similar_artists: return jsonify({"success": True, "artists": [], "source": active_source, "count": 0}) @@ -26915,6 +26771,8 @@ def get_discover_similar_artists(): artist_id = artist.similar_artist_spotify_id elif active_source == 'deezer': artist_id = getattr(artist, 'similar_artist_deezer_id', None) or artist.similar_artist_itunes_id + elif active_source == 'musicbrainz': + artist_id = getattr(artist, 'similar_artist_musicbrainz_id', None) or artist.similar_artist_itunes_id else: artist_id = artist.similar_artist_itunes_id @@ -26922,6 +26780,7 @@ def get_discover_similar_artists(): "artist_id": artist_id, "spotify_artist_id": artist.similar_artist_spotify_id, "itunes_artist_id": artist.similar_artist_itunes_id, + "musicbrainz_artist_id": getattr(artist, 'similar_artist_musicbrainz_id', None), "artist_name": artist.similar_artist_name, "occurrence_count": artist.occurrence_count, "similarity_rank": artist.similarity_rank, @@ -26936,7 +26795,10 @@ def get_discover_similar_artists(): artist_data["popularity"] = artist.popularity result_artists.append(artist_data) - logger.info(f"[Similar Artists] {len(similar_artists)} from DB, {len(result_artists)} valid for {active_source}") + logger.info( + f"[Similar Artists] {len(similar_artists)} from DB, {len(result_artists)} valid for " + f"{active_source} after excluding {active_server} library artists" + ) return jsonify({ "success": True, @@ -26975,6 +26837,8 @@ def enrich_similar_artists(): ext_id = artist.similar_artist_spotify_id elif source == 'deezer': ext_id = getattr(artist, 'similar_artist_deezer_id', None) or artist.similar_artist_itunes_id + elif source == 'musicbrainz': + ext_id = getattr(artist, 'similar_artist_musicbrainz_id', None) or artist.similar_artist_itunes_id else: ext_id = artist.similar_artist_itunes_id if ext_id and ext_id not in cache_map: @@ -28037,6 +27901,118 @@ def get_discovery_shuffle(): logger.error(f"Error getting discovery shuffle playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 + +# ======================================================================== +# Personalized Playlists v2 — unified storage + manager-backed routes. +# Wraps every personalized playlist (Group A + Group B) behind one API +# surface. Generators in `core/personalized/generators/` register at +# import time; this set of routes exposes the manager for the UI. +# Legacy `/api/discover/personalized/...` endpoints stay alive for +# backward compat during the UI migration window. +# ======================================================================== + +# Trigger registration of every generator (side-effect import). +from core.personalized import generators as _personalized_generators # noqa: F401 +from core.personalized import api as _personalized_api +from core.personalized.manager import PersonalizedPlaylistManager as _PersonalizedManager + + +def _build_personalized_manager(): + """Construct a manager wired with whatever each generator needs. + + Per-request construction: the underlying services are cheap + accessors, so we don't bother caching. If profiling shows + overhead, this becomes a module-level lazy singleton.""" + from core.personalized_playlists import get_personalized_playlists_service + from core.seasonal_discovery import get_seasonal_discovery_service + database = get_database() + deps = types.SimpleNamespace( + database=database, + service=get_personalized_playlists_service(database, spotify_client), + seasonal_service=get_seasonal_discovery_service(spotify_client, database), + get_current_profile_id=get_current_profile_id, + get_active_discovery_source=_get_active_discovery_source, + ) + return _PersonalizedManager(database=database, deps=deps) + + +@app.route('/api/personalized/kinds', methods=['GET']) +def personalized_list_kinds(): + """List every registered personalized-playlist kind. Includes the + resolved variant list per kind that supports variants so the UI + can render kind+variant checkboxes without per-kind round-trips.""" + try: + manager = _build_personalized_manager() + return jsonify(_personalized_api.list_kinds(manager=manager)) + except Exception as e: + logger.error(f"Personalized kinds list error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/personalized/playlists', methods=['GET']) +def personalized_list_playlists(): + """List every persisted personalized playlist for the active profile.""" + try: + manager = _build_personalized_manager() + return jsonify(_personalized_api.list_playlists(manager, get_current_profile_id())) + except Exception as e: + logger.error(f"Personalized playlists list error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/personalized/playlist/', methods=['GET']) +@app.route('/api/personalized/playlist//', methods=['GET']) +def personalized_get_playlist(kind, variant=''): + """Get one personalized playlist + its current track snapshot. + + Auto-creates the row from default config if it doesn't exist.""" + try: + manager = _build_personalized_manager() + return jsonify(_personalized_api.get_playlist_with_tracks( + manager, kind, variant, get_current_profile_id(), + )) + except ValueError as e: + return jsonify({"success": False, "error": str(e)}), 400 + except Exception as e: + logger.error(f"Personalized playlist get error ({kind}/{variant}): {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/personalized/playlist//refresh', methods=['POST']) +@app.route('/api/personalized/playlist///refresh', methods=['POST']) +def personalized_refresh_playlist(kind, variant=''): + """Run the kind's generator and persist the snapshot.""" + try: + manager = _build_personalized_manager() + body = request.get_json(silent=True) or {} + overrides = body.get('config_overrides') if isinstance(body.get('config_overrides'), dict) else None + return jsonify(_personalized_api.refresh_playlist( + manager, kind, variant, get_current_profile_id(), config_overrides=overrides, + )) + except ValueError as e: + return jsonify({"success": False, "error": str(e)}), 400 + except Exception as e: + logger.error(f"Personalized playlist refresh error ({kind}/{variant}): {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/personalized/playlist//config', methods=['PUT']) +@app.route('/api/personalized/playlist///config', methods=['PUT']) +def personalized_update_config(kind, variant=''): + """Patch the playlist's per-instance config.""" + try: + manager = _build_personalized_manager() + body = request.get_json(silent=True) or {} + return jsonify(_personalized_api.update_config( + manager, kind, variant, get_current_profile_id(), body, + )) + except ValueError as e: + return jsonify({"success": False, "error": str(e)}), 400 + except Exception as e: + logger.error(f"Personalized playlist config error ({kind}/{variant}): {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + @app.route('/api/discover/artist-blacklist', methods=['GET']) def get_discovery_artist_blacklist(): """Get all blacklisted discovery artists.""" @@ -28756,45 +28732,52 @@ def get_your_artist_info(artist_id): @app.route('/api/image-proxy', methods=['GET']) def image_proxy(): - """Proxy external images to avoid CORS issues for canvas rendering.""" + """Proxy external images to avoid CORS issues for canvas rendering. + + Kept for backwards compatibility; new normalized artwork URLs use + /api/image-cache/, but older browser sessions may still hold this + query-string form. + """ url = request.args.get('url', '') if not url or not url.startswith('http'): return '', 400 - host = urlparse(url).hostname or '' - allowed_hosts = [ - 'i.scdn.co', 'mosaic.scdn.co', # Spotify - 'lastfm.freetls.fastly.net', 'lastfm-img2.akamaized.net', # Last.fm - 'e-cdns-images.dzcdn.net', 'cdns-images.dzcdn.net', 'api.deezer.com', # Deezer - 'is1-ssl.mzstatic.com', 'is2-ssl.mzstatic.com', 'is3-ssl.mzstatic.com', - 'is4-ssl.mzstatic.com', 'is5-ssl.mzstatic.com', # iTunes/Apple - 'img.discogs.com', 'i.discogs.com', # Discogs - 'localhost', '127.0.0.1', 'host.docker.internal', # Local/Docker media servers - ] - if not any(host == h or host.endswith('.' + h) for h in allowed_hosts) and not is_internal_image_host(url): - return '', 403 try: - resp = requests.get(url, timeout=10, stream=True, headers={ - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Referer': 'https://www.deezer.com/', - }) - if resp.status_code != 200: - return '', resp.status_code - content_type = resp.headers.get('Content-Type', 'image/jpeg') - if not content_type.startswith('image/'): - return '', 400 - return Response( - resp.content, - content_type=content_type, - headers={ - 'Cache-Control': 'public, max-age=86400', - 'Access-Control-Allow-Origin': '*', - } - ) - except Exception: + from core.image_cache import get_image_cache + + cached = get_image_cache().get_url(url) + response = send_file(cached.path, mimetype=cached.mime_type, conditional=True) + max_age = int(config_manager.get("image_cache.ttl_seconds", 2592000)) + response.headers['Cache-Control'] = f'private, max-age={max_age}' + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['X-SoulSync-Image-Cache'] = cached.status + return response + except Exception as exc: + logger.debug("image proxy failed: %s", exc) return '', 502 +@app.route('/api/image-cache/', methods=['GET']) +def serve_cached_image(cache_key): + """Serve a registered image URL from SoulSync's disk cache.""" + if not re.fullmatch(r'[a-f0-9]{64}', cache_key or ''): + return '', 404 + + try: + from core.image_cache import get_image_cache + + cached = get_image_cache().get(cache_key) + response = send_file(cached.path, mimetype=cached.mime_type, conditional=True) + max_age = int(config_manager.get("image_cache.ttl_seconds", 2592000)) + response.headers['Cache-Control'] = f'private, max-age={max_age}' + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['X-SoulSync-Image-Cache'] = cached.status + return response + except Exception as exc: + logger.debug("cached image serve failed for %s: %s", cache_key, exc) + return '', 404 + + from core.artists.map import ( _artmap_cache_invalidate, _artmap_cache_get, @@ -33202,6 +33185,20 @@ except Exception as e: # END DEEZER INTEGRATION # ================================================================================================ +amazon_worker = None +try: + amazon_db = MusicDatabase() + amazon_worker = AmazonWorker(database=amazon_db) + amazon_worker.start() + if config_manager.get('amazon_enrichment_paused', False): + amazon_worker.pause() + logger.info("Amazon enrichment worker initialized (paused — restored from config)") + else: + logger.info("Amazon enrichment worker initialized and started") +except Exception as e: + logger.error(f"Amazon worker initialization failed: {e}") + amazon_worker = None + # ================================================================================================ # SPOTIFY ENRICHMENT INTEGRATION @@ -33472,6 +33469,7 @@ _init_service_search( qobuz_worker=qobuz_enrichment_worker, discogs_worker_obj=discogs_worker, audiodb_worker_obj=audiodb_worker, + amazon_worker_obj=amazon_worker, ) # Qobuz status / pause / resume routes are now served by the @@ -33819,6 +33817,101 @@ def stats_recent(): except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 +@app.route('/api/lyrics/fetch', methods=['POST']) +def fetch_lyrics_endpoint(): + """Fetch lyrics for the now-playing media player. + + Body: ``{title, artist, album?, duration?}``. Returns + ``{success, synced, plain, source}`` where ``synced`` is an LRC + string with ``[mm:ss.xx] line`` timestamps (or None) and ``plain`` + is the untimestamped text (or None). ``source`` is the lookup + strategy that hit (``exact`` / ``search`` / ``sidecar``). + + Tries the local ``.lrc`` / ``.txt`` sidecar first when a + ``file_path`` is supplied — already-downloaded tracks should not + bounce LRClib on every play. Falls through to LRClib's exact- + match endpoint when title+artist+album+duration are all available, + then to its generic search endpoint. + """ + try: + data = request.get_json() or {} + title = (data.get('title') or '').strip() + artist = (data.get('artist') or '').strip() + album = (data.get('album') or '').strip() or None + try: + duration = int(data.get('duration') or 0) or None + except (TypeError, ValueError): + duration = None + file_path = data.get('file_path') or None + + if not title or not artist: + return jsonify({'success': False, 'error': 'title and artist required', + 'synced': None, 'plain': None, 'source': None}), 400 + + # 1. Sidecar — fastest, no network. The post-processing flow + # drops .lrc / .txt next to the audio for every successful + # enrichment, so existing downloads almost always have one. + if file_path: + try: + import os as _os + stem, _ = _os.path.splitext(file_path) + lrc_path = stem + '.lrc' + txt_path = stem + '.txt' + if _os.path.exists(lrc_path): + with open(lrc_path, 'r', encoding='utf-8') as fh: + body = fh.read().strip() + if body: + return jsonify({'success': True, 'synced': body, + 'plain': None, 'source': 'sidecar'}) + if _os.path.exists(txt_path): + with open(txt_path, 'r', encoding='utf-8') as fh: + body = fh.read().strip() + if body: + return jsonify({'success': True, 'synced': None, + 'plain': body, 'source': 'sidecar'}) + except Exception as sidecar_err: + logger.debug("lyrics sidecar read skipped: %s", sidecar_err) + + # 2. LRClib network lookup via the shared client instance. + from core.lyrics_client import lyrics_client as _lyrics_client + api = getattr(_lyrics_client, 'api', None) + if api is None: + return jsonify({'success': False, 'error': 'lrclib unavailable', + 'synced': None, 'plain': None, 'source': None}), 200 + + result = None + # Exact-match endpoint requires all four fields. LRClib's API + # will 404 on any miss; treat as soft failure and fall through + # to the search endpoint. + if album and duration: + try: + result = api.get_lyrics(track_name=title, artist_name=artist, + album_name=album, duration=duration) + except Exception as exact_err: + logger.debug("lrclib exact lookup failed: %s", exact_err) + + if result is None: + try: + hits = api.search_lyrics(track_name=title, artist_name=artist) + if hits: + result = hits[0] + except Exception as search_err: + logger.debug("lrclib search lookup failed: %s", search_err) + + if result is None: + return jsonify({'success': False, 'error': 'no lyrics found', + 'synced': None, 'plain': None, 'source': None}) + + synced = getattr(result, 'synced_lyrics', None) or None + plain = getattr(result, 'plain_lyrics', None) or None + return jsonify({'success': bool(synced or plain), 'synced': synced, + 'plain': plain, 'source': 'lrclib'}) + except Exception as e: + logger.error("lyrics fetch failed: %s", e) + return jsonify({'success': False, 'error': str(e), + 'synced': None, 'plain': None, 'source': None}), 500 + + @app.route('/api/stats/resolve-track', methods=['POST']) def stats_resolve_track(): """Resolve a track by title+artist to get its file_path for playback.""" @@ -34264,503 +34357,82 @@ def repair_job_progress(): # IMPORT / STAGING SYSTEM # ================================================================================================ -AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif', '.ape'} +def _build_import_route_runtime(): + return _ImportRouteRuntime( + post_process_matched_download=_post_process_matched_download, + add_activity_item=add_activity_item, + automation_engine=automation_engine, + hydrabase_worker=hydrabase_worker, + dev_mode_enabled=dev_mode_enabled, + import_singles_executor=import_singles_executor, + build_album_import_match_payload=build_album_import_match_payload, + process_single_import_file=lambda runtime, file_info: _process_single_import_file(file_info), + logger=logger, + ) + @app.route('/api/import/staging/files', methods=['GET']) def import_staging_files(): - """Scan the staging folder and return audio files with tag metadata.""" - try: - staging_path = get_staging_path() - os.makedirs(staging_path, exist_ok=True) - - files = [] - for root, _dirs, filenames in os.walk(staging_path): - for fname in filenames: - ext = os.path.splitext(fname)[1].lower() - if ext not in AUDIO_EXTENSIONS: - continue - full_path = os.path.join(root, fname) - rel_path = os.path.relpath(full_path, staging_path) - - meta = read_staging_file_metadata(full_path, rel_path) - - files.append({ - 'filename': fname, - 'rel_path': rel_path, - 'full_path': full_path, - 'title': meta['title'], - 'artist': meta['albumartist'] or meta['artist'] or 'Unknown Artist', - 'album': meta['album'], - 'track_number': meta['track_number'], - 'disc_number': meta['disc_number'], - 'extension': ext - }) - - # Sort by filename - files.sort(key=lambda f: f['filename'].lower()) - return jsonify({'success': True, 'files': files, 'staging_path': staging_path}) - except Exception as e: - logger.error(f"Error scanning staging files: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + payload, status = _import_staging_files(_build_import_route_runtime()) + return jsonify(payload), status @app.route('/api/import/staging/groups', methods=['GET']) def import_staging_groups(): - """Auto-detect album groups from staging files based on their tags. - - Groups files by (album_tag, artist) where both are non-empty and at least 2 files share - the same album+artist combo. Returns groups sorted by file count descending. - """ - try: - staging_path = get_staging_path() - if not os.path.isdir(staging_path): - return jsonify({'success': True, 'groups': []}) - - # Scan files and group by album+artist tags - album_groups = {} # (album_lower, artist_lower) -> {album, artist, files: []} - for root, _dirs, filenames in os.walk(staging_path): - for fname in filenames: - ext = os.path.splitext(fname)[1].lower() - if ext not in AUDIO_EXTENSIONS: - continue - full_path = os.path.join(root, fname) - rel_path = os.path.relpath(full_path, staging_path) - - meta = read_staging_file_metadata(full_path, rel_path) - album = meta['album'] - artist = meta['albumartist'] or meta['artist'] - if not album or not artist: - continue - - key = (album.lower().strip(), artist.lower().strip()) - if key not in album_groups: - album_groups[key] = { - 'album': album.strip(), - 'artist': artist.strip(), - 'files': [] - } - album_groups[key]['files'].append({ - 'filename': fname, - 'full_path': full_path, - 'title': meta['title'], - 'track_number': meta['track_number'], - }) - - # Only return groups with 2+ files - groups = [] - for group in album_groups.values(): - if len(group['files']) >= 2: - group['files'].sort(key=lambda f: f.get('track_number') or 999) - groups.append({ - 'album': group['album'], - 'artist': group['artist'], - 'file_count': len(group['files']), - 'files': group['files'], - 'file_paths': [f['full_path'] for f in group['files']], - }) - - # Sort by file count descending - groups.sort(key=lambda g: g['file_count'], reverse=True) - - return jsonify({'success': True, 'groups': groups}) - except Exception as e: - logger.error(f"Error building staging groups: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + payload, status = _import_staging_groups(_build_import_route_runtime()) + return jsonify(payload), status @app.route('/api/import/staging/hints', methods=['GET']) def import_staging_hints(): - """Extract album search hints from staging folder (tags + folder names). Fast — no Spotify calls.""" - try: - staging_path = get_staging_path() - if not os.path.isdir(staging_path): - return jsonify({'success': True, 'hints': []}) - - # Collect hints from tags and folder structure - tag_albums = {} # (album, artist) -> file count - folder_hints = {} # subfolder name -> file count - - for root, _dirs, filenames in os.walk(staging_path): - audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] - if not audio_files: - continue - - # Folder-based hint: use immediate subfolder name relative to staging - rel_dir = os.path.relpath(root, staging_path) - if rel_dir != '.': - # Use the top-level subfolder as the hint - top_folder = rel_dir.split(os.sep)[0] - folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) - - # Tag-based hints - for fname in audio_files: - full_path = os.path.join(root, fname) - try: - from mutagen import File as MutagenFile - tags = MutagenFile(full_path, easy=True) - if tags: - album = (tags.get('album') or [None])[0] - artist = (tags.get('artist') or (tags.get('albumartist') or [None]))[0] - if album: - key = (album.strip(), (artist or '').strip()) - tag_albums[key] = tag_albums.get(key, 0) + 1 - except Exception as e: - logger.debug("tag read failed: %s", e) - - # Build search queries, prioritizing tag-based hints (more specific) - queries = [] - seen_queries_lower = set() - - # Tag-based: sort by file count descending - for (album, artist), _count in sorted(tag_albums.items(), key=lambda x: -x[1]): - q = f"{album} {artist}".strip() if artist else album - if q.lower() not in seen_queries_lower: - seen_queries_lower.add(q.lower()) - queries.append(q) - - # Folder-based: parse "Artist - Album" pattern or use as-is - for folder, _count in sorted(folder_hints.items(), key=lambda x: -x[1]): - q = folder.replace('_', ' ') - if q.lower() not in seen_queries_lower: - seen_queries_lower.add(q.lower()) - queries.append(q) - - # Cap at 5 queries to keep it fast - queries = queries[:5] - - return jsonify({'success': True, 'hints': queries}) - except Exception as e: - logger.error(f"Error getting staging hints: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + payload, status = _import_staging_hints(_build_import_route_runtime()) + return jsonify(payload), status @app.route('/api/import/search/albums', methods=['GET']) def import_search_albums(): - """Search for albums using the active metadata provider.""" - try: - query = request.args.get('q', '').strip() - if not query: - return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 - - limit = min(int(request.args.get('limit', 12)), 50) - from core.metadata.registry import get_primary_source - - if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: - hydrabase_worker.enqueue(query, 'albums') - - albums = search_import_albums(query, limit=limit) - return jsonify({'success': True, 'albums': albums}) - except Exception as e: - logger.error(f"Error searching albums for import: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + payload, status = _import_search_albums( + _build_import_route_runtime(), + request.args.get('q', ''), + request.args.get('limit', 12), + ) + return jsonify(payload), status @app.route('/api/import/album/match', methods=['POST']) def import_album_match(): - """Match staging files to an album's tracklist.""" - try: - data = request.get_json() or {} - album_id = data.get('album_id') - album_name = data.get('album_name', '') - album_artist = data.get('album_artist', '') - source = str(data.get('source') or '').strip().lower() - # Optional: only match specific files (from auto-group selection) - filter_file_paths = set(data.get('file_paths', [])) - if not album_id: - return jsonify({'success': False, 'error': 'Missing album_id'}), 400 - - # Without `source`, the lookup chain has to guess which metadata - # source the album_id came from — and a Deezer numeric id will - # match nothing in Spotify/iTunes/Discogs/etc., resulting in the - # failure-fallback dict that github issue #524 surfaced as - # "Unknown Artist / album_id-as-title / 0 tracks / 1991". Frontend - # fix in the same PR populates source on every match POST; this - # log catches anything that still reaches us without it (curl, - # third-party, regression in another caller). - if not source: - logger.warning( - "[Import Match] Missing 'source' on album_id=%s — lookup will " - "guess via primary-source priority chain. If this fires " - "consistently, a frontend caller is dropping source from " - "the match POST body.", - album_id, - ) - - payload = build_album_import_match_payload( - album_id, - album_name=album_name, - album_artist=album_artist, - file_paths=filter_file_paths, - source=source or None, - ) - return jsonify(payload) - except Exception as e: - logger.error(f"Error matching album for import: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + payload, status = _import_album_match(_build_import_route_runtime(), request.get_json() or {}) + return jsonify(payload), status @app.route('/api/import/album/process', methods=['POST']) def import_album_process(): - """Process matched album files through the post-processing pipeline.""" - try: - data = request.get_json() or {} - album = data.get('album', {}) - matches = data.get('matches', []) - - if not album or not matches: - return jsonify({'success': False, 'error': 'Missing album or matches data'}), 400 - - processed = 0 - errors = [] - album_name = album.get('name', album.get('album_name', 'Unknown Album')) - artist_name = album.get('artist', album.get('artist_name', 'Unknown Artist')) - album_id = album.get('id', album.get('album_id', '')) - source = str(album.get('source') or data.get('source') or '').strip().lower() - - total_discs = max( - ( - match.get('track', {}).get('disc_number', 1) - for match in matches - if match.get('track') - ), - default=1, - ) - artist_context = resolve_album_artist_context(album, source=source) - - for match in matches: - staging_file = match.get('staging_file') - track = match.get('track') or {} - if not staging_file or not track: - continue - - file_path = staging_file.get('full_path', '') - if not os.path.isfile(file_path): - errors.append(f"File not found: {staging_file.get('filename', '?')}") - continue - - track_name = track.get('name', 'Unknown Track') - track_number = track.get('track_number', 1) - disc_number = track.get('disc_number', 1) - - context_key = f"import_album_{album_id}_{track_number}_{uuid.uuid4().hex[:8]}" - context = build_album_import_context( - album, - track, - artist_context=artist_context, - total_discs=total_discs, - source=source, - ) - - try: - _post_process_matched_download(context_key, context, file_path) - processed += 1 - logger.info(f"Import processed: {track_number}. {track_name} from {album_name}") - except Exception as proc_err: - err_msg = f"{track_name}: {str(proc_err)}" - errors.append(err_msg) - logger.error(f"Import processing error: {err_msg}") - - add_activity_item("", "Album Imported", f"{album_name} by {artist_name} ({processed}/{len(matches)} tracks)", "Now") - - # Emit events through automation engine — same chain as download batches - # batch_complete → auto-scan → library_scan_completed → auto-update DB - if processed > 0: - try: - if automation_engine: - automation_engine.emit('import_completed', { - 'track_count': str(processed), - 'album_name': album_name or '', - 'artist': artist_name or '', - }) - automation_engine.emit('batch_complete', { - 'playlist_name': f"Import: {album_name}" if album_name else 'Import', - 'total_tracks': str(len(matches)), - 'completed_tracks': str(processed), - 'failed_tracks': str(len(errors)), - }) - except Exception as e: - logger.debug("album import automation emit failed: %s", e) - - # Rebuild suggestions cache since staging contents changed - if processed > 0: - refresh_import_suggestions_cache() - - return jsonify({ - 'success': True, - 'processed': processed, - 'total': len(matches), - 'errors': errors - }) - except Exception as e: - logger.error(f"Error processing album import: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + payload, status = _import_album_process(_build_import_route_runtime(), request.get_json() or {}) + return jsonify(payload), status @app.route('/api/import/search/tracks', methods=['GET']) def import_search_tracks(): - """Search tracks using the configured metadata provider priority order.""" - try: - query = request.args.get('q', '').strip() - if not query: - return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 - - limit = min(int(request.args.get('limit', 10)), 30) - if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: - hydrabase_worker.enqueue(query, 'tracks') - - tracks = search_import_tracks(query, limit=limit) - return jsonify({'success': True, 'tracks': tracks}) - except Exception as e: - logger.error(f"Error searching tracks for import: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + payload, status = _import_search_tracks( + _build_import_route_runtime(), + request.args.get('q', ''), + request.args.get('limit', 10), + ) + return jsonify(payload), status def _process_single_import_file(file_info): - """Worker function: validate, resolve metadata, post-process one file. - - Returns ``("ok", title)`` on success, ``("error", message)`` on - failure, or ``("skip", reason)`` for files that need to be reported - but didn't actually run the pipeline. The caller aggregates these. - Designed to be safe to run concurrently from a ThreadPoolExecutor - — each file gets its own UUID context_key, downstream DB writes - serialize via SQLite's busy_timeout, and file-system ops touch - distinct destination paths. - """ - file_path = file_info.get('full_path', '') - if not os.path.isfile(file_path): - return ("error", f"File not found: {file_info.get('filename', '?')}") - - title = file_info.get('title', '') - artist = file_info.get('artist', '') - manual_match = file_info.get('manual_match') - if manual_match is not None and not isinstance(manual_match, dict): - manual_match = None - - manual_match_source = '' - manual_match_id = None - if manual_match: - manual_match_source = str(manual_match.get('source') or '').strip().lower() - manual_match_id = str(manual_match.get('id') or '').strip() - if not manual_match_id or not manual_match_source: - return ("error", f"Malformed manual match for file: {file_info.get('filename', '?')}") - - if not title and not manual_match: - parsed = parse_filename_metadata(file_info.get('filename', '')) - title = parsed.get('title') or os.path.splitext(file_info.get('filename', 'Unknown'))[0] - if not artist: - artist = parsed.get('artist', '') - - from core.imports.resolution import get_single_track_import_context - - try: - resolved = get_single_track_import_context( - title, - artist, - override_id=manual_match_id, - override_source=manual_match_source, - ) - context = normalize_import_context(resolved['context']) - artist_data = get_import_context_artist(context) - track_data = get_import_track_info(context) - final_title = track_data.get('name', title) - final_artist = artist_data.get('name', artist) - - context_key = f"import_single_{uuid.uuid4().hex[:8]}" - _post_process_matched_download(context_key, context, file_path) - logger.info( - "Import single processed: %s by %s (source=%s)", - final_title, - final_artist, - resolved.get('source') or 'local', - ) - return ("ok", final_title) - except Exception as proc_err: - err_msg = f"{title}: {str(proc_err)}" - logger.error(f"Import single processing error: {err_msg}") - return ("error", err_msg) + return _import_process_single_import_file(_build_import_route_runtime(), file_info) @app.route('/api/import/singles/process', methods=['POST']) def import_singles_process(): - """Process individual staging files as singles through the post-processing pipeline. - - Files are processed in parallel through the - ``import_singles_executor`` (3 workers). Per-file work is dominated - by metadata search round-trips, so parallelizing gives a near- - linear speedup without saturating any one provider's rate limits. - """ - try: - data = request.get_json() - files = data.get('files', []) - - if not files: - return jsonify({'success': False, 'error': 'No files provided'}), 400 - - processed = 0 - errors = [] - - # Submit all files at once so the executor pulls 3 at a time. - # as_completed yields in finish order; we don't need ordering - # because the caller just wants a count + error list. - future_to_filename = { - import_singles_executor.submit(_process_single_import_file, file_info): - file_info.get('filename', '?') - for file_info in files - } - - for future in as_completed(future_to_filename): - try: - outcome, payload = future.result() - except Exception as worker_err: - # Catch-all for anything the worker itself didn't catch - # (shouldn't happen — _process_single_import_file wraps - # its own pipeline call — but defensive). - errors.append( - f"{future_to_filename[future]}: worker crashed: {worker_err}" - ) - continue - if outcome == "ok": - processed += 1 - else: - errors.append(payload) - - add_activity_item("", "Singles Imported", f"{processed}/{len(files)} tracks processed", "Now") - - # Emit events through automation engine — same chain as download batches - # batch_complete → auto-scan → library_scan_completed → auto-update DB - if processed > 0: - try: - if automation_engine: - automation_engine.emit('import_completed', { - 'track_count': str(processed), - 'album_name': '', - 'artist': 'Various', - }) - automation_engine.emit('batch_complete', { - 'playlist_name': 'Import: Singles', - 'total_tracks': str(len(files)), - 'completed_tracks': str(processed), - 'failed_tracks': str(len(errors)), - }) - except Exception as e: - logger.debug("singles import automation emit failed: %s", e) - - # Rebuild suggestions cache since staging contents changed - if processed > 0: - refresh_import_suggestions_cache() - - return jsonify({ - 'success': True, - 'processed': processed, - 'total': len(files), - 'errors': errors - }) - except Exception as e: - logger.error(f"Error processing singles import: {e}") - return jsonify({'success': False, 'error': str(e)}), 500 + data = request.get_json() or {} + payload, status = _import_singles_process(_build_import_route_runtime(), data.get('files', [])) + return jsonify(payload), status -# ── Auto-Import Worker ── +# Auto-Import Worker auto_import_worker = None try: from core.auto_import_worker import AutoImportWorker @@ -34942,13 +34614,8 @@ def auto_import_clear_completed(): @app.route('/api/import/staging/suggestions', methods=['GET']) def import_staging_suggestions(): - """Return cached import suggestions. If cache isn't built yet, returns partial/empty with a flag.""" - cache = get_import_suggestions_cache() - return jsonify({ - 'success': True, - 'suggestions': cache['suggestions'], - 'ready': cache['built'], - }) + payload, status = _import_staging_suggestions() + return jsonify(payload), status # ================================================================================================ @@ -35292,6 +34959,11 @@ _register_enrichment_services([ config_paused_key='qobuz_enrichment_paused', extra_status_defaults={'authenticated': False}, ), + _EnrichmentService( + id='amazon', display_name='Amazon Music', + worker_getter=lambda: amazon_worker, + config_paused_key='amazon_enrichment_paused', + ), ]) _configure_enrichment_api( @@ -35311,7 +34983,7 @@ def _emit_rate_monitor_loop(): 'spotify': 'spotify_enrichment', 'itunes': 'itunes_enrichment', 'deezer': 'deezer_enrichment', 'lastfm': 'lastfm', 'genius': 'genius', 'musicbrainz': 'musicbrainz', 'audiodb': 'audiodb', 'discogs': 'discogs', - 'tidal': 'tidal_enrichment', 'qobuz': 'qobuz_enrichment', + 'tidal': 'tidal_enrichment', 'qobuz': 'qobuz_enrichment', 'amazon': 'amazon_enrichment', } while not globals().get('IS_SHUTTING_DOWN', False): @@ -35371,6 +35043,7 @@ def _emit_enrichment_status_loop(): 'genius-enrichment': lambda: genius_worker, 'tidal-enrichment': lambda: tidal_enrichment_worker, 'qobuz-enrichment': lambda: qobuz_enrichment_worker, + 'amazon-enrichment': lambda: amazon_worker, 'hydrabase': lambda: hydrabase_worker, 'soulid': lambda: soulid_worker, 'listening-stats': lambda: listening_stats_worker, @@ -35599,6 +35272,7 @@ def _emit_discovery_progress_loop(): """Push discovery progress to subscribed rooms every 1 second.""" platform_states = { 'tidal': lambda: tidal_discovery_states, + 'qobuz': lambda: qobuz_discovery_states, 'deezer': lambda: deezer_discovery_states, 'youtube': lambda: youtube_playlist_states, 'beatport': lambda: beatport_chart_states, diff --git a/webui/README.md b/webui/README.md index d32587c2..0074b850 100644 --- a/webui/README.md +++ b/webui/README.md @@ -79,6 +79,12 @@ webui/src/ test/ Shared test utilities and setup helpers ``` +Migration planning docs live under `webui/docs/migration/`. + +- keep the high-level route backlog there +- add one route-specific sketch per migration task +- keep migration notes close to the WebUI code rather than the repo root + ### Route Slices - Keep route-specific code inside `webui/src/routes//`. diff --git a/webui/docs/migration/README.md b/webui/docs/migration/README.md new file mode 100644 index 00000000..a7a9d5e3 --- /dev/null +++ b/webui/docs/migration/README.md @@ -0,0 +1,46 @@ +# WebUI Migration Docs + +This folder is the home for React migration planning work inside `webui`. + +## Purpose + +- Keep migration planning close to the code it describes. +- Separate WebUI migration docs from repo-level product or backend docs. +- Give each route migration a predictable place to live. + +## Current Docs + +- [page-migration-overview.md](./page-migration-overview.md) + - high-level route inventory + - migration waves + - cross-route risk assessment +- [stats-migration-plan.md](./stats-migration-plan.md) + - route-specific migration plan for `stats` + +## Naming Guidance + +- Keep one high-level backlog / sequencing doc: + - `page-migration-overview.md` +- Use one route-specific plan per migration task: + - `-migration-plan.md` + +Examples: + +- `search-migration-plan.md` +- `watchlist-migration-plan.md` +- `library-migration-plan.md` + +## Scope + +Use this folder for: + +- migration sequencing +- route-specific implementation sketches +- React ownership cutover notes +- shell handoff notes tied to WebUI page migrations + +Do not use this folder for: + +- generic product docs +- backend architecture notes unrelated to WebUI migration +- permanent user-facing documentation diff --git a/docs/webui-page-migration-analysis.md b/webui/docs/migration/page-migration-overview.md similarity index 87% rename from docs/webui-page-migration-analysis.md rename to webui/docs/migration/page-migration-overview.md index 0d123551..01c96a57 100644 --- a/docs/webui-page-migration-analysis.md +++ b/webui/docs/migration/page-migration-overview.md @@ -1,16 +1,17 @@ -# WebUI Page Migration Analysis +# WebUI Page Migration Overview -Snapshot date: 2026-05-02 +Snapshot date: 2026-05-14 ## Summary - The shell route manifest now has 18 page ids. -- `issues` is still the only React-owned route. +- `issues` and `stats` are now React-owned routes. - Since the last snapshot, the biggest changes are: - `downloads` was renamed into `search`. - The live queue became `active-downloads`. - `watchlist` and `wishlist` became full sidebar pages. - `tools` was split off from `dashboard`. - `artists` is no longer a route id. +- Since the last migration review, `stats` has been fully moved to React, the legacy stats HTML/JS/CSS path has been removed, and the global `Chart.js` import has been dropped in favor of route-local `Recharts`. - The shell is also more modular now. The old monolithic `script.js` has been split across `core.js`, `init.js`, `shared-helpers.js`, and feature modules such as `search.js`, `api-monitor.js`, `pages-extra.js`, `stats-automations.js`, and `wishlist-tools.js`. - Current profile compatibility still normalizes old `downloads` and `artists` references to `search`, so the docs and the route ids are not always using the same historical language. @@ -28,7 +29,7 @@ Snapshot date: 2026-05-02 - `webui/static/core.js` now holds a lot of the shared global state that used to live in the old monolith. - `webui/static/init.js` still owns page activation, permission gating, nav highlighting, legacy routing, and the `window.SoulSyncWebRouter` bridge. - `webui/static/shell-bridge.js` and the TanStack Router adapter still decide whether a route is handled by the React host or handed back to the legacy shell. -- `issues` remains the reference pattern for React-owned pages: route manifest ownership, shell bridge integration, route-local data loading, and detail-modal behavior all live in the React subtree. +- `issues` remains the reference pattern for interactive React-owned pages, while `stats` now complements it as the reference for data-heavy read-only routes with route-local charts and explicit shell handoffs. - The legacy shell is now spread across feature modules rather than one giant coordinator file, which makes the migration seams a little clearer than they were a month ago. ### Route and Compatibility Notes @@ -48,6 +49,7 @@ Snapshot date: 2026-05-02 - Socket/WebSocket and polling behavior remain the biggest migration risks for live pages. - The help system, tours, and helper annotations still reference some historical route names, so route-migration work should use the manifest as the source of truth. - Visual effects such as `particles.js` and `worker-orbs.js` remain shell-global. +- Route migrations should actively scan for emerging shared UI or shell patterns. Do not force abstractions on the first occurrence, but do document overlap and prefer extracting a shared primitive once a second route clearly needs the same behavior. ## Scoring Rubric Each page is scored from 1 to 5 on five axes: @@ -74,10 +76,10 @@ Rollups: | Page | Owner | Scores (R/S/A/C/T) | Effort | Risk | Recommended Wave | | --- | --- | --- | --- | --- | --- | -| `issues` | React | 2 / 2 / 2 / 2 / 2 | Low | Low | Wave 0 | +| `issues` | React | 2 / 2 / 2 / 2 / 2 | Low | Low | Completed | +| `stats` | React | 2 / 2 / 2 / 2 / 2 | Low | Low | Completed | | `help` | Legacy | 3 / 2 / 1 / 1 / 2 | Low | Low | Wave 1 | | `hydrabase` | Legacy | 2 / 2 / 2 / 2 / 2 | Low | Low | Wave 1 | -| `stats` | Legacy | 2 / 2 / 2 / 2 / 2 | Low | Low | Wave 1 | | `import` | Legacy | 3 / 3 / 3 / 2 / 3 | Medium | Medium | Wave 1 | | `search` | Legacy | 4 / 4 / 4 / 3 / 4 | High | High | Wave 2 | | `watchlist` | Legacy | 4 / 4 / 4 / 3 / 4 | High | High | Wave 3 | @@ -104,6 +106,13 @@ Rollups: - Key coupling: shell page gating, shell nav badge refresh, bridge-controlled chrome, React Query cache. - Why it stays first: it is already the canonical React route pattern and the migration baseline. +#### `stats` +- Current owner: React. +- Primary files: `webui/src/routes/stats/*`, `webui/src/platform/shell/*`. +- Main surface: listening stats, charts, ranked lists, disk usage, database storage visualization. +- Key coupling: query invalidation, shell handoffs for playback and artist navigation, route-local chart composition. +- Why it matters now: it is the first completed data-heavy read-only migration and the current reference for route-local charting, explicit shell bridge usage, and legacy cleanup after cutover. + ### Wave 1: Safest wins #### `help` @@ -120,19 +129,12 @@ Rollups: - Key coupling: profile gating and a small amount of shell state. - Recommendation: low-risk route with a narrow surface. -#### `stats` -- Current owner: Legacy. -- Primary files: `webui/index.html`, `webui/static/stats-automations.js`. -- Main surface: listening stats, charts, ranked lists, database storage visualization. -- Key coupling: chart rendering, some deep links back into library routes. -- Recommendation: early migration candidate with good parity-test potential. - #### `import` - Current owner: Legacy. - Primary files: `webui/index.html`, `webui/static/stats-automations.js`, `webui/static/helper.js`. - Main surface: staging files, album and singles matching, suggestion cards, processing queue. - Key coupling: settings-derived staging path assumptions and downstream library state. -- Recommendation: still bounded enough for an early wave, though more workflow-heavy than `help` or `stats`. +- Recommendation: still bounded enough for an early wave, though more workflow-heavy than `help` or `hydrabase`. ### Wave 2: Search split @@ -244,6 +246,7 @@ Rollups: - Recommendation: save this for the final wave with the other complex authoring surfaces. ## Platform Unlocks +- `stats` now provides the baseline for route-local charting, explicit shell-bridge interop from React, and the pattern of cleaning out legacy assets once parity is confirmed. - `search` likely unlocks reusable search-controller and download-launch primitives for the global search widget and other search entry points. - `watchlist` likely unlocks artist-card, per-artist config, and scan-status primitives for `discover` and `wishlist`. - `wishlist` likely unlocks queue/cycle visualization, live polling, and retry-state handling for `active-downloads` and sync-driven download flows. @@ -252,6 +255,7 @@ Rollups: - `library` + `artist-detail` still unlock entity-detail patterns, bulk actions, and file-management workflows. ## Why Earlier Waves Are Safer +- `stats` validated that bounded data pages can move early without needing broad shell rewrites, while also surfacing a healthy reminder to watch for emerging shared primitives during migration work. - Wave 1 routes are either mostly static or bounded data UIs with limited cross-route side effects. - Wave 2 adds the renamed search surface without dragging in the full queue history. - Wave 3 introduces the new watchlist/wishlist split, which is important but still narrower than discovery or library management. @@ -260,8 +264,9 @@ Rollups: - Waves 6-10 defer the broadest, most coupled, or most orchestration-heavy surfaces until the team has the most leverage. ## Final Recommendation -- Keep `issues` as the reference implementation and preserve the existing bridge contract. +- Keep `issues` and `stats` as the current React reference implementations, and preserve the explicit bridge contract between React routes and legacy shell behavior. - Treat `search`, `watchlist`, `wishlist`, `active-downloads`, and `tools` as the current route ids, and keep `downloads` and `artists` only as compatibility history. -- Migrate the safe routes first: `help`, `hydrabase`, `stats`, and `import`. +- Migrate the remaining safe legacy routes first: `help`, `hydrabase`, and `import`. +- During each migration, actively look for small reuse opportunities across route slices and shared UI primitives, but only extract once the overlap is clearly real. - Use `search` as the next meaningful proving ground now that the download queue has been split out. - Avoid pulling `settings`, `sync`, `library`, `artist-detail`, or `automations` forward unless there is a separate product priority strong enough to justify the added regression risk. diff --git a/webui/docs/migration/stats-migration-plan.md b/webui/docs/migration/stats-migration-plan.md new file mode 100644 index 00000000..bd582c2e --- /dev/null +++ b/webui/docs/migration/stats-migration-plan.md @@ -0,0 +1,408 @@ +# WebUI Stats Migration Plan + +Snapshot date: 2026-05-14 + +## Status + +- Completed on 2026-05-14. +- `stats` is now React-owned in the shell route manifest. +- The legacy stats HTML, JS, and CSS path has been removed. +- The global `Chart.js` import was removed and replaced with route-local `Recharts`. +- Legacy playback and artist-detail handoffs now go through the explicit shell bridge. +- A local seed script exists for realistic UI testing without production listening history: `tools/seed_stats_ui_scenarios.py`. + +## Goal + +- Migrate `stats` from the legacy shell to the React route host. +- Replace the global `Chart.js` CDN script with route-local React chart components. +- Use the `issues` route slice as the structural reference, but add a few stronger conventions for data-heavy read-only pages. + +## Why `stats` Is The Right Next Route + +- The route is shell-local today. +- The activation path is narrow. +- The page has real async data loading and interaction. +- The page is complex enough to validate query conventions, search-param state, and route-local chart components. +- The page does not currently drive broad shell-global workflows. + +This route has now validated those assumptions successfully. + +## Current Legacy Shape + +Page surface in `webui/index.html`: + +- Header + - time range buttons + - last synced label + - manual sync action +- Overview cards +- Left column + - listening activity chart + - genre breakdown chart + - recently played list +- Right column + - top artists + - top albums + - top tracks +- Full-width sections + - library health + - library disk usage + - database storage +- Empty state + +Legacy JS responsibilities in `webui/static/stats-automations.js`: + +- page initialization +- range switch handling +- data fetch orchestration +- formatting helpers +- chart instantiation and teardown +- ranked list rendering +- cross-page deep links into library / artist detail +- playback handoff for recent and top tracks + +Backend endpoints already split cleanly: + +- `GET /api/stats/cached` +- `GET /api/stats/db-storage` +- `GET /api/stats/library-disk-usage` +- `POST /api/listening-stats/sync` +- `GET /api/listening-stats/status` + +There are also narrower stats endpoints in the backend, but the current page already gets most of its main payload from the cached route. + +## Library Choice + +Recommended charting library: + +- `recharts` + +Reasoning: + +- React-native component model +- good fit for bar + doughnut-style dashboards +- easy to split into small route-local components +- easier to theme from CSS variables than raw imperative chart setup +- easier to test than a canvas-first imperative path + +Not recommended for this migration: + +- `react-chartjs-2` + - better for parity-only migration + - still keeps the mental model close to Chart.js +- `visx` + - stronger for bespoke visualization systems + - more work than this page needs + +## Proposed Route Slice + +```text +webui/src/routes/stats/ + route.tsx + -stats.types.ts + -stats.api.ts + -stats.helpers.ts + -stats.api.test.ts + -stats.helpers.test.ts + -ui/ + stats-page.tsx + stats-page.module.css + stats-header.tsx + stats-overview-cards.tsx + stats-empty-state.tsx + stats-ranked-list.tsx + stats-recent-plays.tsx + stats-library-health.tsx + stats-disk-usage.tsx + stats-activity-chart.tsx + stats-genre-chart.tsx + stats-db-storage-chart.tsx +``` + +## Proposed Route Responsibilities + +`route.tsx` + +- declare `/stats` +- validate search params +- gate route through `bridge.isPageAllowed('stats')` +- preload the shell context +- load the main cached stats payload plus listening-status payload +- optionally preload disk usage and db storage if we want zero-layout-shift first render + +`-stats.types.ts` + +- search param schema +- response payload types +- normalized display shapes +- chart row types + +`-stats.api.ts` + +- query keys +- fetchers for: + - cached stats + - listening status + - db storage + - library disk usage +- mutation helper for manual sync +- invalidation helpers + +`-stats.helpers.ts` + +- range labels +- numeric and duration formatters +- disk size formatters +- chart data shaping +- legend shaping +- safe fallbacks for empty server responses + +`-ui/stats-page.tsx` + +- page composition +- search-param driven range selection +- section layout +- empty-state branching + +## Search Params + +Use search params for state that should survive reloads and linking: + +- `range` + +Recommended values: + +- `7d` +- `30d` +- `12m` +- `all` + +This is the one clear page-state value worth encoding in the URL. Everything else can remain derived from server data. + +## Query Model + +Recommended split: + +- primary query: + - `statsCachedQueryOptions(profileId, range)` +- secondary queries: + - `statsListeningStatusQueryOptions(profileId)` + - `statsDbStorageQueryOptions(profileId)` + - `statsLibraryDiskUsageQueryOptions(profileId)` + +Why this split: + +- `cached` is the real page backbone +- `db-storage` and `library-disk-usage` are already separate in the backend +- they can render as progressively enhanced cards without blocking the whole route +- `listening-stats/status` updates the sync label and complements the sync mutation + +Recommended route-loader behavior: + +- always ensure: + - cached stats + - listening status +- optional: + - db storage + - disk usage + +If we want a snappier first migration, we should keep the last two as client-side `useQuery` calls rather than route-loader requirements. + +## Component Sketch + +`StatsPage` + +- calls `useReactPageShell('stats')` +- reads `range` from route search +- renders: + - `StatsHeader` + - `StatsOverviewCards` + - `StatsEmptyState` or main sections + +`StatsHeader` + +- range segmented control +- last synced text +- sync button mutation + +`StatsOverviewCards` + +- five summary cards + +`StatsActivityChart` + +- Recharts `BarChart` +- responsive container +- route-local tooltip +- accepts already-shaped rows + +`StatsGenreChart` + +- Recharts `PieChart` +- legend rendered in React markup beside the chart +- top-10 clipping stays in helpers + +`StatsDbStorageChart` + +- Recharts `PieChart` +- custom center label rendered in React +- legend list rendered beside chart + +`StatsRankedList` + +- shared component for artists / albums / tracks +- variant props for: + - artwork + - subtitle/meta + - count label + - optional play action + - optional artist-detail deep link + +`StatsRecentPlays` + +- simple list component +- play action + +`StatsLibraryHealth` + +- overview metrics +- format breakdown bar +- enrichment coverage rows + +`StatsDiskUsage` + +- total bytes row +- pending/deep-scan message +- per-format horizontal bars + +## Recharts Mapping + +Legacy Chart.js to React mapping: + +- listening activity + - from imperative `new Chart(... type: 'bar')` + - to `ResponsiveContainer` + `BarChart` + `Bar` + `XAxis` + `YAxis` + `Tooltip` +- genre breakdown + - from doughnut chart + - to `PieChart` + `Pie` + custom legend +- database storage + - from doughnut chart with center total overlay + - to `PieChart` + `Pie` + React-rendered center label + +Suggested chart convention: + +- keep all chart data shaping outside the chart components +- chart components should receive already-normalized rows and colors +- never read directly from raw server payloads inside Recharts markup + +## CSS Strategy + +Recommended first pass: + +- create `stats-page.module.css` +- port stats-specific selectors from `webui/static/style.css` +- keep class names semantically similar to reduce migration risk + +Suggested approach: + +- move only the selectors needed by the React route +- leave legacy stats selectors in place until the route flip is complete +- after the React route owns `stats`, remove unused legacy selectors in a cleanup pass + +Do not try to redesign the page during the migration. + +## Shell And Routing Changes + +When the route is ready: + +1. Add `webui/src/routes/stats/route.tsx` +2. Regenerate the TanStack route tree if needed +3. Change `stats` from `legacy` to `react` in `webui/src/platform/shell/route-manifest.ts` +4. Keep the legacy `stats-page` DOM in `webui/index.html` during the initial cutover if that reduces risk +5. Remove legacy activation from `webui/static/init.js` once React ownership is confirmed +6. Remove the global Chart.js script from `webui/index.html` + +## Incremental Migration Order + +Recommended order: + +1. Add types, API layer, and helpers +2. Build the React route with plain markup and no charts yet +3. Port overview, ranked lists, recent plays, and empty state +4. Port library health and disk usage +5. Port Recharts activity, genre, and db storage charts +6. Flip route ownership from legacy to React +7. Remove global Chart.js import +8. Delete or shrink legacy `stats` logic from `stats-automations.js` + +This order gives us a working React page before charting becomes the critical path. + +## Testing Sketch + +Unit tests: + +- `-stats.helpers.test.ts` + - range formatting + - duration formatting + - db storage grouping into `Other` + - genre top-10 shaping + - disk usage empty-state shaping + +API tests: + +- `-stats.api.test.ts` + - cached stats success / error + - listening status success / error + - db storage success / error + - disk usage success / error + - sync mutation success / error + +Route / component tests: + +- initial render for default `range=7d` +- changing range updates the URL and query key +- empty state renders when `overview.total_plays === 0` +- ranked artist click deep-links to library / artist detail +- track play action triggers the expected handoff +- sync action shows pending state and invalidates relevant queries + +Playwright is optional for the first pass. + +## Decisions To Keep Simple + +- Keep the existing page structure. +- Keep the current backend endpoint split. +- Keep the current time-range set. +- Reuse the existing shell deep-link behavior for library and playback. +- Use Recharts only inside `stats` first. + +## Follow-Up Opportunities + +- Extract shared chart colors into route-local constants or a small shared viz helper. +- Consider a tiny `components/charts/` layer only after a second React page needs charts. +- Revisit whether `stats/cached` should remain the primary page payload or whether the route should fan out to narrower endpoints later. +- Keep watching for overlap between route-local controls and shared UI primitives. The stats range selector is a good example of a pattern that should stay local for now, but should be reconsidered if another migrated route needs the same segmented-control behavior. + +## Recommendation + +The first implementation should optimize for: + +- parity +- clear route-local boundaries +- removal of global `Chart.js` +- reusable data/query conventions + +It should not optimize for: + +- visual redesign +- a cross-app chart abstraction +- backend reshaping + +## Outcome + +- The route now serves as the reference for data-heavy read-only React pages. +- The migration proved out route-local charts, route-search state, explicit shell-bridge interop, and post-cutover legacy cleanup. +- The work also reinforced a migration guideline for future routes: + - prefer local implementation on first use + - actively note overlap with shared primitives + - extract only once the second clear consumer appears diff --git a/webui/index.html b/webui/index.html index ecb109f9..d590d158 100644 --- a/webui/index.html +++ b/webui/index.html @@ -194,78 +194,78 @@ @@ -541,6 +541,29 @@ + +
+ +
+
+
Amazon Music Enrichment
+
+
Status: Idle +
+
No active matches +
+
Progress: 0 / 0 +
+
+
+
+
-
-

Service Status

-
-
-
- Metadata Source - ● -
-

Disconnected

-

Response: --

-
- -
-
-
-
- Media Server - ● -
-

Disconnected

-

Response: --

-
- -
-
-
-
- Download Source - ● -
-

Disconnected

-

Response: --

-
- -
-
-
- -
+ +
- -
-

Enrichment Services

-
- -
-
- - -
-
- -
- -
-
- -
-
-

Library

-

Checking status...

-
-
- - -
-
- -
+ +
+
+

Your Qobuz Playlists

+ +
+
+
Click 'Refresh' to load your Qobuz playlists.
+
+
+
@@ -2200,6 +2278,7 @@
+
@@ -2353,7 +2432,7 @@
-
@@ -2710,11 +2783,11 @@
- + + @@ -3599,10 +3673,11 @@ +
-
Choose the primary source for artist, album, and track metadata. Spotify can only be selected while an active Spotify session exists. Discogs requires a personal token.
+
Choose the primary source for artist, album, and track metadata. Spotify can only be selected while an active Spotify session exists. Discogs requires a personal token. MusicBrainz is always available but rate-limited to 1 req/sec.
@@ -4151,6 +4226,9 @@ - -
-
-
-
- Stats -

Listening Stats

-
-
-
- - - - -
-
- - -
-
-
- - -
-
-
0
-
Total Plays
-
-
-
0h
-
Listening Time
-
-
-
0
-
Artists
-
-
-
0
-
Albums
-
-
-
0
-
Tracks
-
-
- - -
-
-
-
Listening Activity
-
- -
-
-
-
Genre Breakdown
-
- -
-
-
-
-
Recently Played
-
-
-
-
-
-
Top Artists
-
-
-
-
-
Top Albums
-
-
-
-
Top Tracks
-
-
-
-
- - -
-
Library Health
-
-
-
Format Breakdown
-
-
-
-
0
-
Unplayed Tracks
-
-
-
0h
-
Total Duration
-
-
-
0
-
Total Tracks
-
-
-
-
- - -
-
Library Disk Usage
-
-
-
—
-
Run a Deep Scan to populate
-
-
-
-
- - -
-
Database Storage
-
-
- -
-
-
-
-
- - - -
-
-
@@ -6475,6 +6679,26 @@
+
+
+

Manual Library Match

+
+

Map wishlist and playlist source tracks to library tracks you already own

+
+
+ Source: + Wishlist / Sync +
+
+ Result: + Found +
+
+
+ +
+
+

Retag Tool

@@ -6855,10 +7079,11 @@
@@ -6924,6 +7149,22 @@ + + +
@@ -7566,6 +7807,9 @@ +
@@ -7772,7 +8016,6 @@
- {{ vite_assets('body')|safe }} diff --git a/webui/package-lock.json b/webui/package-lock.json index da9dea53..23b81ab1 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -14,6 +14,7 @@ "ky": "^2.0.2", "react": "^19.2.5", "react-dom": "^19.2.5", + "recharts": "^3.8.1", "zod": "^4.4.2" }, "devDependencies": { @@ -1697,6 +1698,42 @@ "node": ">=18" } }, + "node_modules/@reduxjs/toolkit": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.11.2.tgz", + "integrity": "sha512-Kd6kAHTA6/nUpp8mySPqj3en3dm0tdMIgbttnQ1xFMVpufoj+ADi8pXLBsd4xzTRHQa7t/Jv8W5UnCuW4kuWMQ==", + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@standard-schema/utils": "^0.3.0", + "immer": "^11.0.0", + "redux": "^5.0.1", + "redux-thunk": "^3.1.0", + "reselect": "^5.1.0" + }, + "peerDependencies": { + "react": "^16.9.0 || ^17.0.0 || ^18 || ^19", + "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-redux": { + "optional": true + } + } + }, + "node_modules/@reduxjs/toolkit/node_modules/immer": { + "version": "11.1.8", + "resolved": "https://registry.npmjs.org/immer/-/immer-11.1.8.tgz", + "integrity": "sha512-/tbkHMW7y10Lx6i1crLjD4/OhNkRG+Fo7byZHtah0547nIeXYcpIXaUh0IAQY6gO5459qpGGYapcEOHtFXkIuA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/immer" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.0.0-rc.17", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.17.tgz", @@ -1983,7 +2020,12 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", - "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/utils": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz", + "integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==", "license": "MIT" }, "node_modules/@tanstack/devtools-event-client": { @@ -2412,6 +2454,69 @@ "assertion-error": "^2.0.1" } }, + "node_modules/@types/d3-array": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-shape": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", + "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -2473,6 +2578,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/use-sync-external-store": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz", + "integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==", + "license": "MIT" + }, "node_modules/@vitejs/plugin-react": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.1.tgz", @@ -2954,6 +3065,127 @@ "devOptional": true, "license": "MIT" }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/data-urls": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", @@ -2993,6 +3225,12 @@ "dev": true, "license": "MIT" }, + "node_modules/decimal.js-light": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz", + "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==", + "license": "MIT" + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -3058,6 +3296,16 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/es-toolkit": { + "version": "1.46.1", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.46.1.tgz", + "integrity": "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -3078,6 +3326,12 @@ "@types/estree": "^1.0.0" } }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -3228,6 +3482,16 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/immer": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/immer/-/immer-10.2.0.tgz", + "integrity": "sha512-d/+XTN3zfODyjr89gM3mPq1WNX2B8pYsu7eORitdwyA2sBubnTl3laYlBk4sXY5FUa5qTZGBDPJICVbvqzjlbw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/immer" + } + }, "node_modules/indent-string": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", @@ -3238,6 +3502,15 @@ "node": ">=8" } }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -4116,10 +4389,32 @@ "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", - "dev": true, "license": "MIT", "peer": true }, + "node_modules/react-redux": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", + "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==", + "license": "MIT", + "dependencies": { + "@types/use-sync-external-store": "^0.0.6", + "use-sync-external-store": "^1.4.0" + }, + "peerDependencies": { + "@types/react": "^18.2.25 || ^19", + "react": "^18.0 || ^19", + "redux": "^5.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "redux": { + "optional": true + } + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -4146,6 +4441,36 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/recharts": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-3.8.1.tgz", + "integrity": "sha512-mwzmO1s9sFL0TduUpwndxCUNoXsBw3u3E/0+A+cLcrSfQitSG62L32N69GhqUrrT5qKcAE3pCGVINC6pqkBBQg==", + "license": "MIT", + "workspaces": [ + "www" + ], + "dependencies": { + "@reduxjs/toolkit": "^1.9.0 || 2.x.x", + "clsx": "^2.1.1", + "decimal.js-light": "^2.5.1", + "es-toolkit": "^1.39.3", + "eventemitter3": "^5.0.1", + "immer": "^10.1.1", + "react-redux": "8.x.x || 9.x.x", + "reselect": "5.1.1", + "tiny-invariant": "^1.3.3", + "use-sync-external-store": "^1.2.2", + "victory-vendor": "^37.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -4160,6 +4485,21 @@ "node": ">=8" } }, + "node_modules/redux": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", + "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", + "license": "MIT" + }, + "node_modules/redux-thunk": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-3.1.0.tgz", + "integrity": "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==", + "license": "MIT", + "peerDependencies": { + "redux": "^5.0.0" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -4423,6 +4763,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/tiny-invariant": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", + "license": "MIT" + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -4656,6 +5002,28 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/victory-vendor": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz", + "integrity": "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==", + "license": "MIT AND ISC", + "dependencies": { + "@types/d3-array": "^3.0.3", + "@types/d3-ease": "^3.0.0", + "@types/d3-interpolate": "^3.0.1", + "@types/d3-scale": "^4.0.2", + "@types/d3-shape": "^3.1.0", + "@types/d3-time": "^3.0.0", + "@types/d3-timer": "^3.0.0", + "d3-array": "^3.1.6", + "d3-ease": "^3.0.1", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.1.0", + "d3-time": "^3.0.0", + "d3-timer": "^3.0.1" + } + }, "node_modules/vite": { "version": "8.0.10", "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.10.tgz", diff --git a/webui/package.json b/webui/package.json index 2e076f38..56123f51 100644 --- a/webui/package.json +++ b/webui/package.json @@ -20,6 +20,7 @@ "ky": "^2.0.2", "react": "^19.2.5", "react-dom": "^19.2.5", + "recharts": "^3.8.1", "zod": "^4.4.2" }, "devDependencies": { diff --git a/webui/src/app/api-client.ts b/webui/src/app/api-client.ts index fa54158e..73541181 100644 --- a/webui/src/app/api-client.ts +++ b/webui/src/app/api-client.ts @@ -6,12 +6,21 @@ const apiBaseUrl = typeof globalThis.location === 'object' ? new URL('/api/', globalThis.location.origin).toString() : 'http://localhost/api/'; +const shellBaseUrl = + typeof globalThis.location === 'object' + ? new URL('/', globalThis.location.origin).toString() + : 'http://localhost/'; export const apiClient = ky.create({ baseUrl: apiBaseUrl, retry: 0, }); +export const shellClient = ky.create({ + baseUrl: shellBaseUrl, + retry: 0, +}); + export async function readJson(promise: ResponsePromise): Promise { try { return await promise.json(); diff --git a/webui/src/app/router.test.tsx b/webui/src/app/router.test.tsx index f347a5be..f2d1933c 100644 --- a/webui/src/app/router.test.tsx +++ b/webui/src/app/router.test.tsx @@ -4,31 +4,26 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { SHELL_PROFILE_CONTEXT_CHANGED_EVENT, - type ShellBridge, type ShellProfileContext, type ShellPageId, } from '@/platform/shell/bridge'; +import { HttpResponse, http, server } from '@/test/msw'; +import { createShellBridge } from '@/test/shell-bridge'; import { createAppQueryClient } from './query-client'; import { AppRouterProvider, createAppRouter } from './router'; -function mockIssuesFetch() { - return vi.fn(async (input: RequestInfo | URL) => { - const url = input instanceof Request ? input.url : String(input); - - if (url.includes('/api/issues/counts')) { - return new Response( - JSON.stringify({ +describe('createAppRouter', () => { + beforeEach(() => { + server.use( + http.get('/api/issues/counts', () => + HttpResponse.json({ success: true, counts: { open: 2, in_progress: 1, resolved: 0, dismissed: 0, total: 3 }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, - ); - } - - if (url.includes('/api/issues?')) { - return new Response( - JSON.stringify({ + ), + http.get('/api/issues', () => + HttpResponse.json({ success: true, total: 1, issues: [ @@ -44,30 +39,8 @@ function mockIssuesFetch() { }, ], }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, - ); - } - - throw new Error(`Unexpected fetch request: ${url}`); - }); -} - -function createShellBridge(overrides: Partial = {}): ShellBridge { - return { - getCurrentProfileContext: vi.fn(() => ({ profileId: 1, isAdmin: false })), - isPageAllowed: vi.fn(() => true), - getProfileHomePage: vi.fn<() => ShellPageId>(() => 'discover'), - resolveLegacyPath: vi.fn<(pathname: string) => ShellPageId | null>(() => 'search'), - setActivePageChrome: vi.fn(), - activateLegacyPath: vi.fn(), - showReactHost: vi.fn(), - ...overrides, - }; -} - -describe('createAppRouter', () => { - beforeEach(() => { - vi.stubGlobal('fetch', mockIssuesFetch()); + ), + ); }); afterEach(() => { diff --git a/webui/src/platform/shell/bridge.test.ts b/webui/src/platform/shell/bridge.test.ts index 24787e1b..80b363c2 100644 --- a/webui/src/platform/shell/bridge.test.ts +++ b/webui/src/platform/shell/bridge.test.ts @@ -1,8 +1,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createShellBridge } from '@/test/shell-bridge'; + import type { ShellProfileContext } from './bridge'; -import { SHELL_PROFILE_CONTEXT_CHANGED_EVENT, waitForShellContext } from './bridge'; +import { + SHELL_PROFILE_CONTEXT_CHANGED_EVENT, + bindWindowWebRouter, + waitForShellContext, +} from './bridge'; describe('waitForShellContext', () => { beforeEach(() => { @@ -10,15 +16,7 @@ describe('waitForShellContext', () => { }); it('resolves immediately when the shell already has a profile', async () => { - window.SoulSyncWebShellBridge = { - getProfileHomePage: vi.fn(() => 'discover'), - isPageAllowed: vi.fn(() => true), - activateLegacyPath: vi.fn(), - getCurrentProfileContext: vi.fn(() => ({ profileId: 2, isAdmin: true })), - resolveLegacyPath: vi.fn(() => 'issues'), - setActivePageChrome: vi.fn(), - showReactHost: vi.fn(), - } as NonNullable; + window.SoulSyncWebShellBridge = createShellBridge(); await expect(waitForShellContext()).resolves.toEqual({ bridge: window.SoulSyncWebShellBridge, @@ -31,15 +29,9 @@ describe('waitForShellContext', () => { it('waits for the legacy shell to publish profile context', async () => { const getCurrentProfileContext = vi.fn<() => ShellProfileContext | null>(() => null); - window.SoulSyncWebShellBridge = { - getProfileHomePage: vi.fn(() => 'discover'), - isPageAllowed: vi.fn(() => true), - activateLegacyPath: vi.fn(), + window.SoulSyncWebShellBridge = createShellBridge({ getCurrentProfileContext, - resolveLegacyPath: vi.fn(() => 'issues'), - setActivePageChrome: vi.fn(), - showReactHost: vi.fn(), - } as NonNullable; + }); const contextPromise = waitForShellContext(); @@ -55,3 +47,48 @@ describe('waitForShellContext', () => { }); }); }); + +describe('bindWindowWebRouter', () => { + it('navigates artist detail pages with source-aware URLs', async () => { + const navigate = vi.fn().mockResolvedValue(undefined); + + bindWindowWebRouter({ navigate } as never); + + await window.SoulSyncWebRouter?.navigateToPage('artist-detail', { + artistId: '2YZyLoL8N0Wb9xBt1NhZWg', + artistSource: 'spotify', + }); + + expect(navigate).toHaveBeenCalledWith({ + href: '/artist-detail/spotify/2YZyLoL8N0Wb9xBt1NhZWg', + replace: false, + }); + }); + + it('falls back artist detail URLs to library source when none is supplied', async () => { + const navigate = vi.fn().mockResolvedValue(undefined); + + bindWindowWebRouter({ navigate } as never); + + await window.SoulSyncWebRouter?.navigateToPage('artist-detail', { + artistId: '42', + replace: true, + }); + + expect(navigate).toHaveBeenCalledWith({ + href: '/artist-detail/library/42', + replace: true, + }); + }); + + it('refuses artist detail navigation without an artist id', async () => { + const navigate = vi.fn().mockResolvedValue(undefined); + + bindWindowWebRouter({ navigate } as never); + + await expect( + window.SoulSyncWebRouter?.navigateToPage('artist-detail', {} as never), + ).resolves.toBe(false); + expect(navigate).not.toHaveBeenCalled(); + }); +}); diff --git a/webui/src/platform/shell/bridge.ts b/webui/src/platform/shell/bridge.ts index 04db84c8..cd56f64a 100644 --- a/webui/src/platform/shell/bridge.ts +++ b/webui/src/platform/shell/bridge.ts @@ -1,5 +1,7 @@ import type { AnyRouter } from '@tanstack/react-router'; +import type { ShellStatusPayload } from './status'; + import { getShellRouteByPageId, normalizeShellPath, @@ -17,6 +19,7 @@ export interface ShellProfileContext { export interface ShellContext { bridge: ShellBridge; profile: ShellProfileContext; + status?: ShellStatusPayload | null; } export type ShellBridge = NonNullable; @@ -88,11 +91,18 @@ export function bindWindowWebRouter(router: AnyRouter) { async navigateToPage(pageId, options) { const route = getShellRouteByPageId(pageId); if (!route) return false; + if (pageId === 'artist-detail' && !options?.artistId) { + return false; + } - await router.navigate({ - href: route.path, - replace: options?.replace === true, - }); + let href: `/${string}` = route.path; + if (pageId === 'artist-detail' && options?.artistId) { + const source = options.artistSource ? String(options.artistSource) : 'library'; + href = + `/artist-detail/${encodeURIComponent(source)}/${encodeURIComponent(String(options.artistId))}` as `/${string}`; + } + + await router.navigate({ href, replace: options?.replace === true }); return true; }, }; diff --git a/webui/src/platform/shell/globals.d.ts b/webui/src/platform/shell/globals.d.ts index c7bd15a1..8b043665 100644 --- a/webui/src/platform/shell/globals.d.ts +++ b/webui/src/platform/shell/globals.d.ts @@ -23,6 +23,8 @@ declare global { pageId: ShellPageId, options?: { replace?: boolean; + artistId?: string | number; + artistSource?: string | null; }, ) => Promise; }; @@ -33,7 +35,32 @@ declare global { resolveLegacyPath: (pathname: string) => ShellPageId | null; setActivePageChrome: (pageId: ShellPageId) => void; activateLegacyPath: (pathname: string) => void; + navigateToArtistDetail: ( + artistId: string | number, + artistName: string, + sourceOverride?: string | null, + options?: { + skipRouteChange?: boolean; + }, + ) => void; + cancelSimilarArtistsLoad: () => void; showReactHost: (pageId: ShellPageId) => void; + playLibraryTrack: ( + track: { + id: string | number; + title: string; + file_path: string; + bitrate?: string | number | null; + artist_id?: string | number | null; + album_id?: string | number | null; + _stats_image?: string | null; + }, + albumTitle: string, + artistName: string, + ) => void | Promise; + startStream: (searchResult: Record) => void | Promise; + showLoadingOverlay: (message?: string) => void; + hideLoadingOverlay: () => void; }; } } diff --git a/webui/src/platform/shell/route-controllers.tsx b/webui/src/platform/shell/route-controllers.tsx index a08d3c66..309ed1d1 100644 --- a/webui/src/platform/shell/route-controllers.tsx +++ b/webui/src/platform/shell/route-controllers.tsx @@ -21,6 +21,10 @@ export function useProfile() { return useShellContext().profile; } +export function useShellStatus() { + return useShellContext().status ?? null; +} + export function LegacyRouteController({ pathname }: { pathname: string }) { const bridge = useShellBridge(); diff --git a/webui/src/platform/shell/route-manifest.test.ts b/webui/src/platform/shell/route-manifest.test.ts index 4c384094..2727c255 100644 --- a/webui/src/platform/shell/route-manifest.test.ts +++ b/webui/src/platform/shell/route-manifest.test.ts @@ -6,6 +6,7 @@ import { normalizeShellPath, reactShellRoutes, resolveLegacyShellPageFromPath, + resolveShellNavPage, resolveShellPageFromPath, shellRouteManifest, } from './route-manifest'; @@ -16,7 +17,8 @@ describe('shellRouteManifest', () => { expect(resolveShellPageFromPath('/discover')).toBe('discover'); expect(resolveShellPageFromPath('/watchlist')).toBe('watchlist'); expect(resolveShellPageFromPath('/active-downloads')).toBe('active-downloads'); - expect(resolveShellPageFromPath('/artist-detail')).toBe('artist-detail'); + expect(resolveShellPageFromPath('/artist-detail')).toBeNull(); + expect(resolveShellPageFromPath('/artist-detail/spotify/2YZyLoL8N0Wb9xBt1NhZWg')).toBe('artist-detail'); expect(resolveShellPageFromPath('/artists')).toBeNull(); }); @@ -40,8 +42,9 @@ describe('shellRouteManifest', () => { it('tracks whether a route is rendered by React or the legacy shell', () => { expect(getShellRouteByPageId('issues')?.kind).toBe('react'); + expect(getShellRouteByPageId('stats')?.kind).toBe('react'); expect(getShellRouteByPageId('discover')?.kind).toBe('legacy'); - expect(reactShellRoutes.map((route) => route.pageId)).toEqual(['issues']); + expect(reactShellRoutes.map((route) => route.pageId)).toEqual(['stats', 'issues']); expect(legacyShellRoutes.some((route) => route.pageId === 'dashboard')).toBe(true); }); @@ -49,7 +52,14 @@ describe('shellRouteManifest', () => { expect(resolveLegacyShellPageFromPath('/search')).toBe('search'); expect(resolveLegacyShellPageFromPath('/active-downloads')).toBe('active-downloads'); expect(resolveLegacyShellPageFromPath('/tools')).toBe('tools'); + expect(resolveLegacyShellPageFromPath('/artist-detail')).toBeNull(); + expect(resolveLegacyShellPageFromPath('/artist-detail/deezer/12345')).toBe('artist-detail'); expect(resolveLegacyShellPageFromPath('/issues')).toBeNull(); expect(resolveLegacyShellPageFromPath('/does-not-exist')).toBeNull(); }); + + it('maps contextual pages to the nav chrome they belong under', () => { + expect(resolveShellNavPage('artist-detail')).toBe('library'); + expect(resolveShellNavPage('stats')).toBe('stats'); + }); }); diff --git a/webui/src/platform/shell/route-manifest.ts b/webui/src/platform/shell/route-manifest.ts index 1f07f582..bbcd7d09 100644 --- a/webui/src/platform/shell/route-manifest.ts +++ b/webui/src/platform/shell/route-manifest.ts @@ -42,7 +42,7 @@ export const shellRouteManifest: readonly ShellRouteDefinition[] = [ { pageId: 'library', path: '/library', kind: 'legacy' }, { pageId: 'tools', path: '/tools', kind: 'legacy' }, { pageId: 'artist-detail', path: '/artist-detail', kind: 'legacy' }, - { pageId: 'stats', path: '/stats', kind: 'legacy' }, + { pageId: 'stats', path: '/stats', kind: 'react' }, { pageId: 'settings', path: '/settings', kind: 'legacy' }, { pageId: 'issues', path: '/issues', kind: 'react' }, { pageId: 'help', path: '/help', kind: 'legacy' }, @@ -71,10 +71,29 @@ export function getShellRouteByPath(pathname: string): ShellRouteDefinition | un } export function resolveShellPageFromPath(pathname: string): ShellPageId | null { + const normalized = normalizeShellPath(pathname); + if (normalized === '/artist-detail') { + return null; + } + if (normalized.startsWith('/artist-detail/')) { + return 'artist-detail'; + } return getShellRouteByPath(pathname)?.pageId ?? null; } export function resolveLegacyShellPageFromPath(pathname: string): ShellPageId | null { + const normalized = normalizeShellPath(pathname); + if (normalized === '/artist-detail') { + return null; + } + if (normalized.startsWith('/artist-detail/')) { + return 'artist-detail'; + } const route = getShellRouteByPath(pathname); return route?.kind === 'legacy' ? route.pageId : null; } + +export function resolveShellNavPage(pageId: ShellPageId): ShellPageId | '' { + if (pageId === 'artist-detail') return 'library'; + return pageId; +} diff --git a/webui/src/platform/shell/status.test.ts b/webui/src/platform/shell/status.test.ts new file mode 100644 index 00000000..68c1bf6f --- /dev/null +++ b/webui/src/platform/shell/status.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; + +import { HttpResponse, http, server } from '@/test/msw'; + +import { fetchShellStatus } from './status'; + +describe('shell status', () => { + it('fetches the shell status payload', async () => { + server.use( + http.get('/status', () => + HttpResponse.json({ media_server: { type: 'plex', connected: true } }), + ), + ); + + await expect(fetchShellStatus()).resolves.toEqual({ + media_server: { type: 'plex', connected: true }, + }); + }); +}); diff --git a/webui/src/platform/shell/status.ts b/webui/src/platform/shell/status.ts new file mode 100644 index 00000000..47b53321 --- /dev/null +++ b/webui/src/platform/shell/status.ts @@ -0,0 +1,25 @@ +import { queryOptions } from '@tanstack/react-query'; + +import { readJson, shellClient } from '@/app/api-client'; + +export interface ShellStatusMediaServer { + type?: string | null; + connected?: boolean | null; +} + +export interface ShellStatusPayload { + media_server?: ShellStatusMediaServer | null; +} + +export async function fetchShellStatus(): Promise { + return await readJson(shellClient.get('status')); +} + +export function shellStatusQueryOptions() { + return queryOptions({ + queryKey: ['shell', 'status'] as const, + queryFn: fetchShellStatus, + staleTime: 30_000, + retry: false, + }); +} diff --git a/webui/src/routeTree.gen.ts b/webui/src/routeTree.gen.ts index d90bb0ce..addc76fb 100644 --- a/webui/src/routeTree.gen.ts +++ b/webui/src/routeTree.gen.ts @@ -10,14 +10,21 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as SplatRouteImport } from './routes/$' +import { Route as StatsRouteRouteImport } from './routes/stats/route' import { Route as IssuesRouteRouteImport } from './routes/issues/route' import { Route as IndexRouteImport } from './routes/index' +import { Route as ArtistDetailSourceIdRouteImport } from './routes/artist-detail/$source/$id' const SplatRoute = SplatRouteImport.update({ id: '/$', path: '/$', getParentRoute: () => rootRouteImport, } as any) +const StatsRouteRoute = StatsRouteRouteImport.update({ + id: '/stats', + path: '/stats', + getParentRoute: () => rootRouteImport, +} as any) const IssuesRouteRoute = IssuesRouteRouteImport.update({ id: '/issues', path: '/issues', @@ -28,35 +35,54 @@ const IndexRoute = IndexRouteImport.update({ path: '/', getParentRoute: () => rootRouteImport, } as any) +const ArtistDetailSourceIdRoute = ArtistDetailSourceIdRouteImport.update({ + id: '/artist-detail/$source/$id', + path: '/artist-detail/$source/$id', + getParentRoute: () => rootRouteImport, +} as any) export interface FileRoutesByFullPath { '/': typeof IndexRoute '/issues': typeof IssuesRouteRoute + '/stats': typeof StatsRouteRoute '/$': typeof SplatRoute + '/artist-detail/$source/$id': typeof ArtistDetailSourceIdRoute } export interface FileRoutesByTo { '/': typeof IndexRoute '/issues': typeof IssuesRouteRoute + '/stats': typeof StatsRouteRoute '/$': typeof SplatRoute + '/artist-detail/$source/$id': typeof ArtistDetailSourceIdRoute } export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute '/issues': typeof IssuesRouteRoute + '/stats': typeof StatsRouteRoute '/$': typeof SplatRoute + '/artist-detail/$source/$id': typeof ArtistDetailSourceIdRoute } export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath - fullPaths: '/' | '/issues' | '/$' + fullPaths: '/' | '/issues' | '/stats' | '/$' | '/artist-detail/$source/$id' fileRoutesByTo: FileRoutesByTo - to: '/' | '/issues' | '/$' - id: '__root__' | '/' | '/issues' | '/$' + to: '/' | '/issues' | '/stats' | '/$' | '/artist-detail/$source/$id' + id: + | '__root__' + | '/' + | '/issues' + | '/stats' + | '/$' + | '/artist-detail/$source/$id' fileRoutesById: FileRoutesById } export interface RootRouteChildren { IndexRoute: typeof IndexRoute IssuesRouteRoute: typeof IssuesRouteRoute + StatsRouteRoute: typeof StatsRouteRoute SplatRoute: typeof SplatRoute + ArtistDetailSourceIdRoute: typeof ArtistDetailSourceIdRoute } declare module '@tanstack/react-router' { @@ -68,6 +94,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof SplatRouteImport parentRoute: typeof rootRouteImport } + '/stats': { + id: '/stats' + path: '/stats' + fullPath: '/stats' + preLoaderRoute: typeof StatsRouteRouteImport + parentRoute: typeof rootRouteImport + } '/issues': { id: '/issues' path: '/issues' @@ -82,13 +115,22 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof IndexRouteImport parentRoute: typeof rootRouteImport } + '/artist-detail/$source/$id': { + id: '/artist-detail/$source/$id' + path: '/artist-detail/$source/$id' + fullPath: '/artist-detail/$source/$id' + preLoaderRoute: typeof ArtistDetailSourceIdRouteImport + parentRoute: typeof rootRouteImport + } } } const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, IssuesRouteRoute: IssuesRouteRoute, + StatsRouteRoute: StatsRouteRoute, SplatRoute: SplatRoute, + ArtistDetailSourceIdRoute: ArtistDetailSourceIdRoute, } export const routeTree = rootRouteImport ._addFileChildren(rootRouteChildren) diff --git a/webui/src/routes/__root.tsx b/webui/src/routes/__root.tsx index d46454e1..44575f1c 100644 --- a/webui/src/routes/__root.tsx +++ b/webui/src/routes/__root.tsx @@ -3,13 +3,18 @@ import { Outlet, createRootRouteWithContext } from '@tanstack/react-router'; import type { AppRouterContext } from '@/app/router'; import { waitForShellContext } from '@/platform/shell/bridge'; +import { shellStatusQueryOptions } from '@/platform/shell/status'; import { IssueDomainHost } from './issues/-ui/issue-domain-host'; export const Route = createRootRouteWithContext()({ - beforeLoad: async () => { - const shell = await waitForShellContext(); - return { shell }; + beforeLoad: async ({ context }) => { + const [shell, status] = await Promise.all([ + waitForShellContext(), + context.queryClient.fetchQuery(shellStatusQueryOptions()).catch(() => undefined), + ]); + + return { shell: { ...shell, status } }; }, component: () => ( <> diff --git a/webui/src/routes/artist-detail/$source/$id.tsx b/webui/src/routes/artist-detail/$source/$id.tsx new file mode 100644 index 00000000..6987c00e --- /dev/null +++ b/webui/src/routes/artist-detail/$source/$id.tsx @@ -0,0 +1,31 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { useLayoutEffect } from 'react'; + +import { useShellBridge } from '@/platform/shell/route-controllers'; + +export const Route = createFileRoute('/artist-detail/$source/$id')({ + component: ArtistDetailPage, +}); + +// Thin legacy handoff: TanStack owns the URL shape here, but the vanilla JS +// artist-detail page still renders the actual experience for now. The route +// owns cancellation so similar-artist loading stops when this page changes. +function ArtistDetailPage() { + const bridge = useShellBridge(); + const { source, id } = Route.useParams(); + + useLayoutEffect(() => { + if (!bridge) return; + + const normalizedSource = source.toLowerCase() === 'library' ? null : source.toLowerCase(); + bridge.navigateToArtistDetail(id, '', normalizedSource, { + skipRouteChange: true, + }); + + return () => { + bridge.cancelSimilarArtistsLoad(); + }; + }, [bridge, id, source]); + + return null; +} diff --git a/webui/src/routes/artist-detail/-route.test.tsx b/webui/src/routes/artist-detail/-route.test.tsx new file mode 100644 index 00000000..c1426ba9 --- /dev/null +++ b/webui/src/routes/artist-detail/-route.test.tsx @@ -0,0 +1,84 @@ +import { createMemoryHistory } from '@tanstack/react-router'; +import { render, waitFor } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAppQueryClient } from '@/app/query-client'; +import { AppRouterProvider, createAppRouter } from '@/app/router'; +import { createShellBridge } from '@/test/shell-bridge'; + +function renderArtistDetailRoute(initialEntries = ['/artist-detail/library/42']) { + const queryClient = createAppQueryClient(); + const history = createMemoryHistory({ initialEntries }); + const router = createAppRouter({ history, queryClient }); + + return { + history, + router, + ...render(), + }; +} + +describe('artist-detail route', () => { + beforeEach(() => { + window.SoulSyncWebShellBridge = createShellBridge(); + }); + + afterEach(() => { + window.SoulSyncWebShellBridge = undefined; + }); + + it('hands off canonical artist-detail URLs to the legacy shell', async () => { + renderArtistDetailRoute(['/artist-detail/spotify/2YZyLoL8N0Wb9xBt1NhZWg']); + + await waitFor(() => { + expect(window.SoulSyncWebShellBridge?.navigateToArtistDetail).toHaveBeenCalledWith( + '2YZyLoL8N0Wb9xBt1NhZWg', + '', + 'spotify', + { + skipRouteChange: true, + }, + ); + }); + }); + + it('normalizes library sources before handing off', async () => { + renderArtistDetailRoute(['/artist-detail/library/42']); + + await waitFor(() => { + expect(window.SoulSyncWebShellBridge?.navigateToArtistDetail).toHaveBeenCalledWith( + '42', + '', + null, + { + skipRouteChange: true, + }, + ); + }); + }); + + it('cancels the similar artists stream when the route unmounts', async () => { + const { unmount } = renderArtistDetailRoute(['/artist-detail/spotify/2YZyLoL8N0Wb9xBt1NhZWg']); + + await waitFor(() => { + expect(window.SoulSyncWebShellBridge?.navigateToArtistDetail).toHaveBeenCalledWith( + '2YZyLoL8N0Wb9xBt1NhZWg', + '', + 'spotify', + { + skipRouteChange: true, + }, + ); + }); + + const cancelSimilarArtistsLoad = window.SoulSyncWebShellBridge + ?.cancelSimilarArtistsLoad as ReturnType; + cancelSimilarArtistsLoad.mockClear(); + + unmount(); + + await waitFor(() => { + expect(cancelSimilarArtistsLoad).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/webui/src/routes/issues/-route.test.tsx b/webui/src/routes/issues/-route.test.tsx index 402966e1..7a0abdcd 100644 --- a/webui/src/routes/issues/-route.test.tsx +++ b/webui/src/routes/issues/-route.test.tsx @@ -2,10 +2,9 @@ import { createMemoryHistory } from '@tanstack/react-router'; import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { ShellBridge, ShellPageId } from '@/platform/shell/bridge'; - import { createAppQueryClient } from '@/app/query-client'; import { AppRouterProvider, createAppRouter } from '@/app/router'; +import { createShellBridge } from '@/test/shell-bridge'; function createResponse(body: unknown, ok = true, status = 200) { return new Response(JSON.stringify(body), { @@ -14,19 +13,6 @@ function createResponse(body: unknown, ok = true, status = 200) { }); } -function createShellBridge(overrides: Partial = {}): ShellBridge { - return { - getCurrentProfileContext: vi.fn(() => ({ profileId: 2, isAdmin: true })), - isPageAllowed: vi.fn(() => true), - getProfileHomePage: vi.fn<() => ShellPageId>(() => 'discover'), - resolveLegacyPath: vi.fn<(pathname: string) => ShellPageId | null>(() => 'search'), - setActivePageChrome: vi.fn(), - activateLegacyPath: vi.fn(), - showReactHost: vi.fn(), - ...overrides, - }; -} - function renderIssuesRoute(initialEntries = ['/issues']) { const queryClient = createAppQueryClient(); const history = createMemoryHistory({ initialEntries }); diff --git a/webui/src/routes/stats/-route.test.tsx b/webui/src/routes/stats/-route.test.tsx new file mode 100644 index 00000000..ec251ae9 --- /dev/null +++ b/webui/src/routes/stats/-route.test.tsx @@ -0,0 +1,169 @@ +import { createMemoryHistory } from '@tanstack/react-router'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAppQueryClient } from '@/app/query-client'; +import { AppRouterProvider, createAppRouter } from '@/app/router'; +import { HttpResponse, http, server } from '@/test/msw'; +import { createShellBridge } from '@/test/shell-bridge'; + +function renderStatsRoute(initialEntries = ['/stats']) { + const queryClient = createAppQueryClient(); + const history = createMemoryHistory({ initialEntries }); + const router = createAppRouter({ history, queryClient }); + + return { + history, + ...render(), + }; +} + +describe('stats route', () => { + beforeEach(() => { + window.SoulSyncWebShellBridge = createShellBridge(); + window.showToast = vi.fn(); + server.use( + http.get('/api/stats/cached', () => + HttpResponse.json({ + success: true, + overview: { + total_plays: 24, + total_time_ms: 6_600_000, + unique_artists: 3, + unique_albums: 4, + unique_tracks: 12, + }, + top_artists: [{ id: 7, name: 'Artist A', play_count: 10 }], + top_albums: [], + top_tracks: [], + timeline: [{ date: 'May 10', plays: 4 }], + genres: [{ genre: 'House', play_count: 10, percentage: 80 }], + recent: [{ title: 'Track A', artist: 'Artist A', played_at: '2026-05-14T08:00:00Z' }], + health: { total_tracks: 12, format_breakdown: { FLAC: 12 } }, + }), + ), + http.get('/api/listening-stats/status', () => + HttpResponse.json({ stats: { last_poll: '2026-05-14 10:00:00' } }), + ), + http.get('/status', () => + HttpResponse.json({ media_server: { type: 'plex', connected: true } }), + ), + http.get('/api/stats/db-storage', () => + HttpResponse.json({ + success: true, + tables: [{ name: 'tracks', size: 2048 }], + total_file_size: 4096, + method: 'dbstat', + }), + ), + http.get('/api/stats/library-disk-usage', () => + HttpResponse.json({ + success: true, + has_data: true, + total_bytes: 2048, + tracks_with_size: 12, + tracks_without_size: 0, + by_format: { flac: 2048 }, + }), + ), + ); + }); + + it('renders the stats page through the app router', async () => { + renderStatsRoute(); + + await waitFor(() => expect(screen.getByTestId('stats-page')).toBeInTheDocument()); + expect(await screen.findByText('Listening Stats')).toBeInTheDocument(); + expect(screen.getByText('24')).toBeInTheDocument(); + expect(window.SoulSyncWebShellBridge?.showReactHost).toHaveBeenCalledWith('stats'); + expect(window.SoulSyncWebShellBridge?.setActivePageChrome).toHaveBeenCalledWith('stats'); + }); + + it('still renders when listening stats status prefetch fails', async () => { + server.use( + http.get('/api/listening-stats/status', () => + HttpResponse.json({ error: 'status unavailable' }, { status: 500 }), + ), + ); + + renderStatsRoute(); + + await waitFor(() => expect(screen.getByTestId('stats-page')).toBeInTheDocument()); + expect(await screen.findByText('Listening Stats')).toBeInTheDocument(); + expect(screen.getByText('Not synced yet')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Sync listening stats' })).toBeInTheDocument(); + }); + + it('shows an explicit standalone notice instead of the sync button', async () => { + server.use( + http.get('/status', () => + HttpResponse.json({ media_server: { type: 'soulsync', connected: true } }), + ), + ); + + renderStatsRoute(); + + await waitFor(() => expect(screen.getByTestId('stats-page')).toBeInTheDocument()); + expect(await screen.findByText('Listening Stats')).toBeInTheDocument(); + expect(screen.getByText('Standalone mode: manual sync unavailable')).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: 'Sync listening stats' })).not.toBeInTheDocument(); + }); + + it('stores the time range in route search state', async () => { + const { history } = renderStatsRoute(); + + fireEvent.click(await screen.findByRole('button', { name: '30 Days' })); + + await waitFor(() => expect(history.location.search).toContain('range=30d')); + }); + + it('hands artist detail navigation directly to the shell bridge', async () => { + renderStatsRoute(); + + fireEvent.click(await screen.findByRole('button', { name: 'Artist A' })); + + expect(window.SoulSyncWebShellBridge?.navigateToArtistDetail).toHaveBeenCalledWith( + 7, + 'Artist A', + ); + }); + + it('falls back to streaming when track resolution fails', async () => { + window.SoulSyncWebShellBridge = createShellBridge({ + startStream: vi.fn(), + }); + + server.use( + http.post('/api/stats/resolve-track', () => + HttpResponse.json({ error: 'resolve unavailable' }, { status: 500 }), + ), + http.post('/api/enhanced-search/stream-track', () => + HttpResponse.json({ + success: true, + result: { stream_url: '/api/stream/1' }, + }), + ), + ); + + renderStatsRoute(); + + fireEvent.click((await screen.findAllByTitle('Play'))[0]); + + await waitFor(() => + expect(window.SoulSyncWebShellBridge?.startStream).toHaveBeenCalledWith({ + stream_url: '/api/stream/1', + }), + ); + expect(window.SoulSyncWebShellBridge?.playLibraryTrack).not.toHaveBeenCalled(); + }); + + it('redirects back home when the page is not allowed', async () => { + window.SoulSyncWebShellBridge = createShellBridge({ + isPageAllowed: vi.fn((pageId) => pageId !== 'stats'), + }); + + const { history } = renderStatsRoute(['/stats']); + + await waitFor(() => expect(history.location.pathname).toBe('/discover')); + }); +}); diff --git a/webui/src/routes/stats/-stats.api.test.ts b/webui/src/routes/stats/-stats.api.test.ts new file mode 100644 index 00000000..ab85853e --- /dev/null +++ b/webui/src/routes/stats/-stats.api.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest'; + +import { HttpResponse, http, server } from '@/test/msw'; + +import { + fetchListeningStatsStatus, + fetchStatsCached, + fetchStatsDbStorage, + fetchStatsLibraryDiskUsage, + resolveStatsTrack, + streamStatsTrack, + triggerListeningStatsSync, +} from './-stats.api'; + +describe('stats api', () => { + it('fetches the cached stats payload for a range', async () => { + server.use( + http.get('/api/stats/cached', ({ request }) => { + const url = new URL(request.url); + expect(url.searchParams.get('range')).toBe('30d'); + + return HttpResponse.json({ + success: true, + overview: { total_plays: 12 }, + top_artists: [], + top_albums: [], + top_tracks: [], + timeline: [], + genres: [], + recent: [], + health: {}, + }); + }), + ); + + await expect(fetchStatsCached('30d')).resolves.toMatchObject({ + overview: { total_plays: 12 }, + }); + }); + + it('returns an empty payload when cached stats are not available yet', async () => { + server.use( + http.get('/api/stats/cached', () => + HttpResponse.json({ success: false, error: 'Listening stats not synced yet' }), + ), + ); + + await expect(fetchStatsCached('7d')).resolves.toMatchObject({ + success: true, + overview: { total_plays: 0 }, + top_artists: [], + top_albums: [], + top_tracks: [], + timeline: [], + genres: [], + recent: [], + health: {}, + }); + }); + + it('returns an empty payload when the server reports a cache miss as an HTTP error', async () => { + server.use( + http.get('/api/stats/cached', () => + HttpResponse.json({ error: 'No cached stats available yet' }, { status: 500 }), + ), + ); + + await expect(fetchStatsCached('7d')).resolves.toMatchObject({ + success: true, + overview: { total_plays: 0 }, + top_artists: [], + top_albums: [], + top_tracks: [], + timeline: [], + genres: [], + recent: [], + health: {}, + }); + }); + + it('surfaces db storage and disk usage errors', async () => { + server.use( + http.get('/api/stats/db-storage', () => + HttpResponse.json({ error: 'db unavailable' }, { status: 500 }), + ), + http.get('/api/stats/library-disk-usage', () => + HttpResponse.json({ error: 'disk unavailable' }, { status: 500 }), + ), + ); + + await expect(fetchStatsDbStorage()).rejects.toThrow('db unavailable'); + await expect(fetchStatsLibraryDiskUsage()).rejects.toThrow('disk unavailable'); + }); + + it('reads listening status and triggers manual sync', async () => { + server.use( + http.get('/api/listening-stats/status', () => + HttpResponse.json({ stats: { last_poll: '2026-05-14 10:00:00' } }), + ), + http.post('/api/listening-stats/sync', () => HttpResponse.json({ success: true })), + ); + + await expect(fetchListeningStatsStatus()).resolves.toEqual({ + stats: { last_poll: '2026-05-14 10:00:00' }, + }); + await expect(triggerListeningStatsSync()).resolves.toBeUndefined(); + }); + + it('resolves and streams tracks through the stats playback helpers', async () => { + server.use( + http.post('/api/stats/resolve-track', async ({ request }) => { + await expect(request.json()).resolves.toEqual({ + title: 'Track', + artist: 'Artist', + }); + return HttpResponse.json({ + success: true, + track: { id: 1, title: 'Track', file_path: '/music/track.flac' }, + }); + }), + http.post('/api/enhanced-search/stream-track', async ({ request }) => { + await expect(request.json()).resolves.toEqual({ + track_name: 'Track', + artist_name: 'Artist', + album_name: 'Album', + duration_ms: 0, + }); + return HttpResponse.json({ + success: true, + result: { stream_url: '/api/stream/1' }, + }); + }), + ); + + await expect(resolveStatsTrack('Track', 'Artist')).resolves.toMatchObject({ + id: 1, + title: 'Track', + }); + await expect(streamStatsTrack('Track', 'Artist', 'Album')).resolves.toEqual({ + stream_url: '/api/stream/1', + }); + }); +}); diff --git a/webui/src/routes/stats/-stats.api.ts b/webui/src/routes/stats/-stats.api.ts new file mode 100644 index 00000000..e0514233 --- /dev/null +++ b/webui/src/routes/stats/-stats.api.ts @@ -0,0 +1,157 @@ +import { queryOptions, type QueryClient } from '@tanstack/react-query'; +import { HTTPError } from 'ky'; + +import { apiClient, readJson } from '@/app/api-client'; + +import type { + ListeningStatsStatus, + StatsCachedPayload, + StatsDbStoragePayload, + StatsLibraryDiskUsagePayload, + StatsRange, + StatsResolveTrackPayload, + StatsStreamTrackPayload, +} from './-stats.types'; + +import { EMPTY_STATS_PAYLOAD } from './-stats.helpers'; + +export const STATS_QUERY_KEY = ['stats'] as const; + +const NO_STATS_YET_PATTERNS = [ + /not synced/i, + /no listening stats/i, + /no cached stats/i, + /cache miss/i, + /stats cache.*(missing|empty|not found)/i, +] as const; + +function isNoStatsYetMessage(message: string | undefined): boolean { + if (!message) return false; + return NO_STATS_YET_PATTERNS.some((pattern) => pattern.test(message)); +} + +function getEmptyStatsPayload(): StatsCachedPayload { + return { + success: true, + ...EMPTY_STATS_PAYLOAD, + }; +} + +export async function fetchStatsCached(range: StatsRange): Promise { + try { + const payload = await readJson( + apiClient.get('stats/cached', { + searchParams: { range }, + }), + ); + if (!payload.success) { + if (isNoStatsYetMessage(payload.error)) { + return getEmptyStatsPayload(); + } + throw new Error(payload.error || 'Failed to load listening stats'); + } + return payload; + } catch (error) { + if (error instanceof HTTPError && isNoStatsYetMessage(error.message)) { + return getEmptyStatsPayload(); + } + throw error; + } +} + +export async function fetchListeningStatsStatus(): Promise { + return await readJson(apiClient.get('listening-stats/status')); +} + +export async function fetchStatsDbStorage(): Promise { + const payload = await readJson(apiClient.get('stats/db-storage')); + if (!payload.success) { + throw new Error(payload.error || 'Failed to load database storage'); + } + return payload; +} + +export async function fetchStatsLibraryDiskUsage(): Promise { + const payload = await readJson( + apiClient.get('stats/library-disk-usage'), + ); + if (!payload.success) { + throw new Error(payload.error || 'Failed to load library disk usage'); + } + return payload; +} + +export async function triggerListeningStatsSync(): Promise { + const payload = await readJson<{ success: boolean; error?: string }>( + apiClient.post('listening-stats/sync'), + ); + if (!payload.success) { + throw new Error(payload.error || 'Sync failed'); + } +} + +export async function resolveStatsTrack( + title: string, + artist: string, +): Promise { + const payload = await readJson( + apiClient.post('stats/resolve-track', { + json: { title, artist }, + }), + ); + if (!payload.success) return null; + return payload.track ?? null; +} + +export async function streamStatsTrack( + title: string, + artist: string, + album: string, +): Promise | null> { + const payload = await readJson( + apiClient.post('enhanced-search/stream-track', { + json: { + track_name: title, + artist_name: artist, + album_name: album, + duration_ms: 0, + }, + }), + ); + if (!payload.success) { + throw new Error(payload.error || 'Track not found in library or any source'); + } + return payload.result ?? null; +} + +export function statsCachedQueryOptions(range: StatsRange) { + return queryOptions({ + queryKey: [...STATS_QUERY_KEY, 'cached', range], + queryFn: () => fetchStatsCached(range), + }); +} + +export function listeningStatsStatusQueryOptions() { + return queryOptions({ + queryKey: [...STATS_QUERY_KEY, 'listening-status'], + queryFn: fetchListeningStatsStatus, + }); +} + +export function statsDbStorageQueryOptions() { + return queryOptions({ + queryKey: [...STATS_QUERY_KEY, 'db-storage'], + queryFn: fetchStatsDbStorage, + }); +} + +export function statsLibraryDiskUsageQueryOptions() { + return queryOptions({ + queryKey: [...STATS_QUERY_KEY, 'library-disk-usage'], + queryFn: fetchStatsLibraryDiskUsage, + }); +} + +export function invalidateStatsQueries(queryClient: QueryClient) { + return queryClient.invalidateQueries({ queryKey: STATS_QUERY_KEY }); +} diff --git a/webui/src/routes/stats/-stats.helpers.test.ts b/webui/src/routes/stats/-stats.helpers.test.ts new file mode 100644 index 00000000..0814546f --- /dev/null +++ b/webui/src/routes/stats/-stats.helpers.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; + +import { + formatBytes, + formatDbStorageValue, + formatListeningTime, + formatRelativePlayedAt, + getTopArtistBubbles, + groupDbStorageTables, + hasStatsData, +} from './-stats.helpers'; +import { statsSearchSchema } from './-stats.types'; + +describe('statsSearchSchema', () => { + it('falls back to 7d for unknown ranges', () => { + expect(statsSearchSchema.parse({ range: 'bad' })).toEqual({ range: '7d' }); + }); + + it('keeps known ranges', () => { + expect(statsSearchSchema.parse({ range: '12m' })).toEqual({ range: '12m' }); + }); +}); + +describe('stats helpers', () => { + it('detects whether the page has listening data', () => { + expect(hasStatsData({ total_plays: 0 })).toBe(false); + expect(hasStatsData({ total_plays: 4 })).toBe(true); + }); + + it('formats listening time and bytes', () => { + expect(formatListeningTime(3_900_000)).toBe('1h 5m'); + expect(formatBytes(2_097_152)).toBe('2.00 MB'); + }); + + it('formats relative recent-play times', () => { + const now = new Date('2026-05-14T12:00:00.000Z').getTime(); + expect(formatRelativePlayedAt('2026-05-14T11:15:00.000Z', now)).toBe('45m ago'); + expect(formatRelativePlayedAt('2026-05-14T08:00:00.000Z', now)).toBe('4h ago'); + }); + + it('groups db storage rows into Other after the top eight', () => { + const grouped = groupDbStorageTables( + Array.from({ length: 10 }, (_, index) => ({ + name: `table_${index + 1}`, + size: index + 1, + })), + ); + + expect(grouped).toHaveLength(9); + expect(grouped.at(-1)).toEqual({ name: 'Other', size: 19 }); + }); + + it('formats db storage by method', () => { + expect(formatDbStorageValue(2_097_152, 'dbstat')).toBe('2.0 MB'); + expect(formatDbStorageValue(1240, 'rowcount')).toBe('1,240 rows'); + }); + + it('shapes top artist bubbles from the highest-play artist', () => { + const bubbles = getTopArtistBubbles([ + { name: 'A', play_count: 20 }, + { name: 'B', play_count: 10 }, + ]); + + expect(bubbles[0]?.percent).toBe(100); + expect(bubbles[1]?.percent).toBe(50); + }); +}); diff --git a/webui/src/routes/stats/-stats.helpers.ts b/webui/src/routes/stats/-stats.helpers.ts new file mode 100644 index 00000000..d20d9742 --- /dev/null +++ b/webui/src/routes/stats/-stats.helpers.ts @@ -0,0 +1,162 @@ +import type { + StatsArtistRow, + StatsCachedPayload, + StatsDbStorageTable, + StatsHealth, + StatsOverview, + StatsRange, +} from './-stats.types'; + +export const EMPTY_STATS_OVERVIEW: StatsOverview = { + total_plays: 0, + total_time_ms: 0, + unique_artists: 0, + unique_albums: 0, + unique_tracks: 0, +}; + +export const EMPTY_STATS_PAYLOAD: Required< + Pick< + StatsCachedPayload, + 'overview' | 'top_artists' | 'top_albums' | 'top_tracks' | 'timeline' | 'genres' | 'recent' + > +> & { health: StatsHealth } = { + overview: EMPTY_STATS_OVERVIEW, + top_artists: [], + top_albums: [], + top_tracks: [], + timeline: [], + genres: [], + recent: [], + health: {}, +}; + +export const STATS_GENRE_COLORS = [ + '#1db954', + '#1ed760', + '#4ade80', + '#7c3aed', + '#a855f7', + '#ec4899', + '#f43f5e', + '#f97316', + '#eab308', + '#06b6d4', +] as const; + +export const STATS_DB_STORAGE_COLORS = [ + '#3b82f6', + '#f97316', + '#a855f7', + '#14b8a6', + '#eab308', + '#ec4899', + '#6366f1', + '#22c55e', + '#555555', +] as const; + +export const STATS_ENRICHMENT_SERVICES = [ + { key: 'spotify', label: 'Spotify', color: '#1db954' }, + { key: 'musicbrainz', label: 'MusicBrainz', color: '#ba55d3' }, + { key: 'deezer', label: 'Deezer', color: '#a238ff' }, + { key: 'lastfm', label: 'Last.fm', color: '#d51007' }, + { key: 'itunes', label: 'iTunes', color: '#fc3c44' }, + { key: 'audiodb', label: 'AudioDB', color: '#1a9fff' }, + { key: 'genius', label: 'Genius', color: '#ffff64' }, + { key: 'tidal', label: 'Tidal', color: '#00ffff' }, + { key: 'qobuz', label: 'Qobuz', color: '#4285f4' }, +] as const; + +export function getStatsRangeLabel(range: StatsRange): string { + switch (range) { + case '7d': + return '7 Days'; + case '30d': + return '30 Days'; + case '12m': + return '12 Months'; + case 'all': + return 'All Time'; + } +} + +export function hasStatsData(overview: Partial | undefined): boolean { + return (overview?.total_plays ?? 0) > 0; +} + +export function formatCompactNumber(value: number | null | undefined): string { + if (!value) return '0'; + if (value >= 1_000_000) return `${stripTrailingZero((value / 1_000_000).toFixed(1))}M`; + if (value >= 1_000) return `${stripTrailingZero((value / 1_000).toFixed(1))}K`; + return value.toLocaleString(); +} + +export function formatListeningTime(totalMs: number | null | undefined): string { + if (!totalMs) return '0h'; + const hours = Math.floor(totalMs / 3_600_000); + const minutes = Math.floor((totalMs % 3_600_000) / 60_000); + return hours > 0 ? `${hours}h ${minutes}m` : `${minutes}m`; +} + +export function formatTotalDuration(totalMs: number | null | undefined): string { + if (!totalMs) return '0h'; + return `${Math.floor(totalMs / 3_600_000)}h`; +} + +export function formatRelativePlayedAt( + dateStr: string | null | undefined, + now = Date.now(), +): string { + if (!dateStr) return ''; + const diff = now - new Date(dateStr).getTime(); + const minutes = Math.floor(diff / 60_000); + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `${hours}h ago`; + const days = Math.floor(hours / 24); + if (days < 30) return `${days}d ago`; + return `${Math.floor(days / 30)}mo ago`; +} + +export function formatBytes(value: number | null | undefined): string { + if (!value || value <= 0) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let index = 0; + let next = value; + while (next >= 1024 && index < units.length - 1) { + next /= 1024; + index += 1; + } + return `${next.toFixed(next < 10 ? 2 : 1)} ${units[index]}`; +} + +export function groupDbStorageTables(tables: StatsDbStorageTable[]): StatsDbStorageTable[] { + const top = tables.slice(0, 8); + const rest = tables.slice(8); + const restSize = rest.reduce((sum, table) => sum + table.size, 0); + return restSize > 0 ? [...top, { name: 'Other', size: restSize }] : top; +} + +export function formatDbStorageValue(size: number, method: string | null | undefined): string { + if (method === 'dbstat') { + if (size > 1_048_576) return `${(size / 1_048_576).toFixed(1)} MB`; + return `${Math.round(size / 1024)} KB`; + } + return `${size.toLocaleString()} rows`; +} + +export function getTopArtistBubbles(artists: StatsArtistRow[]) { + const top = artists.slice(0, 5); + const maxPlays = top[0]?.play_count || 1; + + return top.map((artist, index) => ({ + artist, + percent: Math.round((artist.play_count / maxPlays) * 100), + size: 44 + (4 - index) * 6, + })); +} + +function stripTrailingZero(value: string): string { + return value.replace(/\.0$/, ''); +} diff --git a/webui/src/routes/stats/-stats.types.ts b/webui/src/routes/stats/-stats.types.ts new file mode 100644 index 00000000..c00cd25a --- /dev/null +++ b/webui/src/routes/stats/-stats.types.ts @@ -0,0 +1,148 @@ +import { z } from 'zod'; + +export const STATS_RANGE_VALUES = ['7d', '30d', '12m', 'all'] as const; +export type StatsRange = (typeof STATS_RANGE_VALUES)[number]; + +export const statsSearchSchema = z.object({ + range: z.enum(STATS_RANGE_VALUES).default('7d').catch('7d'), +}); + +export type StatsSearch = z.infer; + +export interface StatsOverview { + total_plays: number; + total_time_ms: number; + unique_artists: number; + unique_albums: number; + unique_tracks: number; +} + +export interface StatsArtistRow { + id?: string | number | null; + name: string; + image_url?: string | null; + play_count: number; + global_listeners?: number | null; + soul_id?: string | null; +} + +export interface StatsAlbumRow { + name: string; + artist?: string | null; + artist_id?: string | number | null; + image_url?: string | null; + play_count: number; +} + +export interface StatsTrackRow { + name: string; + artist?: string | null; + artist_id?: string | number | null; + album?: string | null; + image_url?: string | null; + play_count: number; +} + +export interface StatsTimelineRow { + date: string; + plays: number; +} + +export interface StatsGenreRow { + genre: string; + play_count: number; + percentage: number; +} + +export interface StatsEnrichmentCoverage { + spotify?: number; + musicbrainz?: number; + deezer?: number; + lastfm?: number; + itunes?: number; + audiodb?: number; + genius?: number; + tidal?: number; + qobuz?: number; +} + +export interface StatsHealth { + total_tracks?: number; + unplayed_count?: number; + unplayed_percentage?: number; + total_duration_ms?: number; + format_breakdown?: Record; + enrichment_coverage?: StatsEnrichmentCoverage; +} + +export interface StatsRecentTrack { + title: string; + artist?: string | null; + album?: string | null; + played_at?: string | null; +} + +export interface StatsCachedPayload { + success: boolean; + overview?: Partial; + top_artists?: StatsArtistRow[]; + top_albums?: StatsAlbumRow[]; + top_tracks?: StatsTrackRow[]; + timeline?: StatsTimelineRow[]; + genres?: StatsGenreRow[]; + recent?: StatsRecentTrack[]; + health?: StatsHealth; + error?: string; +} + +export interface ListeningStatsStatus { + stats?: { + last_poll?: string | null; + }; + error?: string; +} + +export interface StatsDbStorageTable { + name: string; + size: number; +} + +export interface StatsDbStoragePayload { + success: boolean; + tables?: StatsDbStorageTable[]; + total_file_size?: number; + method?: string; + error?: string; +} + +export interface StatsLibraryDiskUsagePayload { + success: boolean; + has_data?: boolean; + total_bytes?: number; + tracks_with_size?: number; + tracks_without_size?: number; + by_format?: Record; + error?: string; +} + +export interface StatsResolveTrackPayload { + success: boolean; + error?: string; + track?: { + id: string | number; + title: string; + file_path: string; + bitrate?: string | number | null; + artist_id?: string | number | null; + album_id?: string | number | null; + image_url?: string | null; + album_title?: string | null; + artist_name?: string | null; + }; +} + +export interface StatsStreamTrackPayload { + success: boolean; + error?: string; + result?: Record; +} diff --git a/webui/src/routes/stats/-ui/stats-page.module.css b/webui/src/routes/stats/-ui/stats-page.module.css new file mode 100644 index 00000000..17be6814 --- /dev/null +++ b/webui/src/routes/stats/-ui/stats-page.module.css @@ -0,0 +1,938 @@ +.statsContainer { + margin: 20px; + padding: 28px 24px 30px; + overflow: hidden; + background: linear-gradient(135deg, rgba(20, 20, 20, 0.55) 0%, rgba(12, 12, 12, 0.62) 100%); + border-radius: 24px; + border: 1px solid rgba(255, 255, 255, 0.08); + border-top: 1px solid rgba(255, 255, 255, 0.12); + box-shadow: + 0 8px 32px rgba(0, 0, 0, 0.3), + 0 4px 16px rgba(0, 0, 0, 0.2), + inset 0 1px 0 rgba(255, 255, 255, 0.08); +} + +.statsHeader { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + margin: -28px -24px 20px; + padding: 20px 24px; + min-height: 176px; + background: linear-gradient( + 180deg, + rgba(var(--accent-rgb), 0.1) 0%, + rgba(var(--accent-rgb), 0.04) 40%, + transparent 100% + ); + border-bottom: 1px solid rgba(255, 255, 255, 0.06); + border-top-left-radius: 24px; + border-top-right-radius: 24px; + position: relative; + overflow: hidden; + flex-wrap: wrap; +} + +.statsHeader::before { + content: ''; + position: absolute; + inset: 0; + background: linear-gradient( + 90deg, + transparent 0%, + rgba(var(--accent-rgb), 0.08) 50%, + transparent 100% + ); + transform: translateX(-100%); + animation: headerSweep 12s ease-in-out infinite; +} + +@keyframes headerSweep { + 50% { + transform: translateX(100%); + } + 100% { + transform: translateX(100%); + } +} + +.statsHeaderTitle { + display: flex; + align-items: center; + gap: 14px; + position: relative; + z-index: 1; +} + +.headerIcon { + width: 176px; + height: 176px; + object-fit: contain; + flex-shrink: 0; + filter: drop-shadow(0 2px 8px rgba(var(--accent-rgb), 0.3)); + transition: + transform 0.3s ease, + filter 0.3s ease; +} + +.headerIcon:hover { + transform: scale(1.08) rotate(-3deg); + filter: drop-shadow(0 4px 14px rgba(var(--accent-rgb), 0.5)); +} + +.headerTitle { + margin: 0; + font-size: 28px; + font-weight: 700; + color: #fff; +} + +.statsHeaderControls { + display: flex; + align-items: center; + gap: 16px; + position: relative; + z-index: 1; + min-width: 0; + flex-wrap: wrap; + justify-content: flex-end; +} + +.statsTimeRange { + display: flex; + gap: 4px; + padding: 3px; + border-radius: 10px; + background: rgba(255, 255, 255, 0.04); + border: 1px solid rgba(255, 255, 255, 0.06); +} + +.statsRangeButton { + padding: 7px 16px; + border: none; + border-radius: 8px; + background: transparent; + color: rgba(255, 255, 255, 0.5); + font-size: 0.82rem; + font-weight: 600; + cursor: pointer; + transition: all 0.2s ease; + font-family: inherit; +} + +.statsRangeButton:hover { + color: rgba(255, 255, 255, 0.8); + background: rgba(255, 255, 255, 0.04); +} + +.statsRangeButtonActive { + background: rgb(var(--accent-rgb)); + color: #fff; + box-shadow: 0 2px 8px rgba(var(--accent-rgb), 0.3); +} + +.statsSyncControls { + display: flex; + align-items: center; + gap: 8px; + min-width: 0; +} + +.statsLastSynced { + flex: 1 1 auto; + min-width: 0; + font-size: 0.72rem; + color: rgba(255, 255, 255, 0.3); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.statsStandaloneNotice { + max-width: 260px; + font-size: 0.72rem; + line-height: 1.35; + color: rgba(255, 255, 255, 0.38); + text-align: right; +} + +.statsSyncButton { + width: 32px; + height: 32px; + border-radius: 8px; + border: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.04); + color: rgba(255, 255, 255, 0.5); + font-size: 16px; + cursor: pointer; + transition: all 0.2s ease; + display: flex; + align-items: center; + justify-content: center; +} + +.statsSyncButton:hover { + background: rgba(255, 255, 255, 0.08); + color: #fff; + border-color: rgba(255, 255, 255, 0.15); +} + +.statsSyncButtonSyncing { + pointer-events: none; + color: transparent; + position: relative; +} + +.statsSyncButtonSyncing::after { + content: ''; + position: absolute; + width: 14px; + height: 14px; + border: 2px solid rgba(var(--accent-rgb), 0.2); + border-top-color: rgba(var(--accent-rgb), 0.8); + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} + +.statsOverview { + display: grid; + grid-template-columns: repeat(5, 1fr); + gap: 14px; + margin-bottom: 20px; +} + +.statsCard { + background: linear-gradient(135deg, rgba(20, 20, 20, 0.95) 0%, rgba(12, 12, 12, 0.98) 100%); + border: 1px solid rgba(255, 255, 255, 0.08); + border-radius: 14px; + padding: 20px; + text-align: center; + position: relative; + overflow: hidden; + transition: all 0.3s ease; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.3); +} + +.statsCard::before { + content: ''; + position: absolute; + top: 0; + left: 20%; + right: 20%; + height: 2px; + background: linear-gradient(90deg, transparent, rgba(var(--accent-rgb), 0.5), transparent); +} + +.statsCard:hover { + transform: translateY(-3px); + border-color: rgba(var(--accent-rgb), 0.2); + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.4); +} + +.statsCardValue { + font-size: 2rem; + font-weight: 700; + color: #fff; + line-height: 1.2; + margin-bottom: 6px; +} + +.statsCardLabel { + font-size: 0.78rem; + color: rgba(255, 255, 255, 0.45); + text-transform: uppercase; + letter-spacing: 0.06em; + font-weight: 600; +} + +.statsMainGrid { + display: grid; + grid-template-columns: 1fr 360px; + gap: 20px; + min-width: 0; + margin-bottom: 20px; +} + +.statsLeftCol, +.statsRightCol { + display: flex; + flex-direction: column; + gap: 20px; + min-width: 0; +} + +.statsSectionCard { + background: rgba(255, 255, 255, 0.02); + border: 1px solid rgba(255, 255, 255, 0.06); + border-radius: 14px; + padding: 20px; + transition: border-color 0.2s ease; + min-width: 0; + overflow: hidden; + margin-bottom: 20px; +} + +.statsSectionCard:hover { + border-color: rgba(255, 255, 255, 0.1); +} + +.statsSectionTitle { + font-size: 0.78rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: rgba(255, 255, 255, 0.4); + margin-bottom: 16px; + padding-bottom: 10px; + border-bottom: 1px solid rgba(255, 255, 255, 0.05); +} + +.chartContainer { + position: relative; + height: 220px; +} + +.statsGenreChartContainer { + display: flex; + align-items: center; + gap: 24px; +} + +.statsGenreChartWrap { + width: 180px; + height: 180px; + flex-shrink: 0; +} + +.statsGenreLegend { + flex: 1; + display: flex; + flex-direction: column; + gap: 6px; +} + +.statsGenreLegendItem { + display: flex; + align-items: center; + gap: 8px; + font-size: 0.82rem; + color: rgba(255, 255, 255, 0.7); +} + +.statsGenreDot, +.statsDbLegendDot { + width: 10px; + height: 10px; + border-radius: 3px; + flex-shrink: 0; +} + +.statsGenrePct { + margin-left: auto; + color: rgba(255, 255, 255, 0.4); + font-variant-numeric: tabular-nums; +} + +.statsTopArtistsVisual { + margin-bottom: 16px; + padding-bottom: 14px; + border-bottom: 1px solid rgba(255, 255, 255, 0.04); +} + +.statsArtistBubbles { + display: flex; + justify-content: space-around; + align-items: flex-end; + gap: 8px; +} + +.statsArtistBubble { + display: flex; + flex-direction: column; + align-items: center; + gap: 6px; + min-width: 0; + flex: 1; + background: transparent; + border: none; + padding: 0; + cursor: pointer; + transition: transform 0.2s ease; +} + +.statsArtistBubble:disabled { + cursor: default; +} + +.statsArtistBubble:hover { + transform: translateY(-3px); +} + +.statsBubbleImage { + border-radius: 50%; + background-size: cover; + background-position: center; + background-color: rgba(255, 255, 255, 0.06); + border: 2px solid rgba(var(--accent-rgb), 0.2); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.3); + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; +} + +.statsBubbleInitial { + font-size: 1.2rem; + font-weight: 700; + color: rgba(255, 255, 255, 0.4); +} + +.statsBubbleBarContainer { + width: 100%; + height: 3px; + background: rgba(255, 255, 255, 0.06); + border-radius: 2px; + overflow: hidden; +} + +.statsBubbleBar { + height: 100%; + background: linear-gradient(90deg, rgb(var(--accent-rgb)), rgba(var(--accent-rgb), 0.4)); + border-radius: 2px; +} + +.statsBubbleName { + font-size: 0.7rem; + color: rgba(255, 255, 255, 0.7); + font-weight: 500; + text-align: center; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 100%; +} + +.statsBubbleCount { + font-size: 0.65rem; + color: rgba(var(--accent-rgb), 0.7); + font-weight: 600; +} + +.statsRankedList, +.statsRecentList { + display: flex; + flex-direction: column; + gap: 4px; + max-height: 300px; + overflow-y: auto; + scrollbar-width: thin; + scrollbar-color: rgba(255, 255, 255, 0.1) transparent; +} + +.statsRankedItem, +.statsRecentItem { + display: flex; + align-items: center; + gap: 10px; + padding: 8px 10px; + border-radius: 8px; + transition: background 0.15s ease; +} + +.statsRankedItem:hover, +.statsRecentItem:hover { + background: rgba(255, 255, 255, 0.04); +} + +.statsRankedNum { + font-size: 0.75rem; + color: rgba(255, 255, 255, 0.25); + font-weight: 700; + width: 18px; + text-align: right; + flex-shrink: 0; +} + +.statsRankedImage, +.statsRankedImageFallback { + width: 36px; + height: 36px; + border-radius: 6px; + flex-shrink: 0; + background: rgba(255, 255, 255, 0.05); + object-fit: cover; +} + +.statsRankedInfo { + flex: 1; + min-width: 0; +} + +.statsRankedName { + font-size: 0.88rem; + color: rgba(255, 255, 255, 0.85); + font-weight: 500; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + display: flex; + align-items: center; + gap: 6px; +} + +.statsRankedMeta { + font-size: 0.72rem; + color: rgba(255, 255, 255, 0.4); +} + +.statsRankedCount { + font-size: 0.78rem; + color: rgba(var(--accent-rgb), 0.8); + font-weight: 600; + flex-shrink: 0; + font-variant-numeric: tabular-nums; +} + +.statsArtistLink { + color: inherit; + text-decoration: none; + background: transparent; + border: none; + padding: 0; + cursor: pointer; + font: inherit; +} + +.statsArtistLink:hover { + color: rgb(var(--accent-rgb)); +} + +.statsSoulIdBadge { + width: 12px; + height: 12px; + opacity: 0.5; +} + +.statsPlayButton { + width: 28px; + height: 28px; + border-radius: 50%; + border: none; + background: rgba(var(--accent-rgb), 0.15); + color: rgb(var(--accent-rgb)); + font-size: 10px; + cursor: pointer; + flex-shrink: 0; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s ease; + opacity: 0; +} + +.statsRankedItem:hover .statsPlayButton, +.statsRecentItem:hover .statsPlayButton { + opacity: 1; +} + +.statsPlayButton:hover { + background: rgb(var(--accent-rgb)); + color: #fff; + transform: scale(1.1); +} + +.statsPlayButtonSmall { + width: 22px; + height: 22px; + font-size: 8px; +} + +.statsRecentTitle { + flex: 1; + font-size: 0.85rem; + color: rgba(255, 255, 255, 0.8); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.statsRecentArtist { + font-size: 0.78rem; + color: rgba(255, 255, 255, 0.4); + flex-shrink: 0; +} + +.statsRecentTime { + font-size: 0.72rem; + color: rgba(255, 255, 255, 0.25); + flex-shrink: 0; + min-width: 65px; + text-align: right; +} + +.statsHealthGrid { + display: grid; + grid-template-columns: 2fr 1fr 1fr 1fr; + gap: 16px; + align-items: start; +} + +.statsHealthItem { + text-align: center; +} + +.statsHealthItemWide { + text-align: left; +} + +.statsHealthValue { + font-size: 1.6rem; + font-weight: 700; + color: #fff; + line-height: 1.2; + margin-bottom: 4px; +} + +.statsHealthLabel { + font-size: 0.75rem; + color: rgba(255, 255, 255, 0.4); + text-transform: uppercase; + letter-spacing: 0.04em; + font-weight: 600; +} + +.statsFormatBar { + display: flex; + height: 28px; + border-radius: 8px; + overflow: hidden; + margin-top: 8px; + background: rgba(255, 255, 255, 0.04); +} + +.statsFormatSegment { + display: flex; + align-items: center; + justify-content: center; + font-size: 0.68rem; + font-weight: 600; + color: #fff; + white-space: nowrap; + min-width: 30px; +} + +.statsEnrichment { + display: flex; + gap: 16px; + margin-top: 16px; + padding-top: 14px; + border-top: 1px solid rgba(255, 255, 255, 0.04); + flex-wrap: wrap; +} + +.statsEnrichItem { + flex: 1; + min-width: 120px; + display: flex; + align-items: center; + gap: 8px; +} + +.statsEnrichName { + font-size: 0.72rem; + color: rgba(255, 255, 255, 0.45); + min-width: 70px; + font-weight: 500; +} + +.statsEnrichBar { + flex: 1; + height: 4px; + background: rgba(255, 255, 255, 0.06); + border-radius: 2px; + overflow: hidden; +} + +.statsEnrichFill { + height: 100%; + border-radius: 2px; +} + +.statsEnrichPct { + font-size: 0.72rem; + color: rgba(255, 255, 255, 0.4); + font-variant-numeric: tabular-nums; + min-width: 30px; + text-align: right; +} + +.statsDiskUsageWrap { + display: flex; + flex-direction: column; + gap: 14px; + margin-top: 8px; +} + +.statsDiskTotalRow { + display: flex; + align-items: baseline; + gap: 16px; + flex-wrap: wrap; +} + +.statsDiskTotalValue { + font-size: 28px; + font-weight: 700; + color: rgb(var(--accent-rgb)); +} + +.statsDiskTotalMeta { + font-size: 12px; + color: rgba(255, 255, 255, 0.55); +} + +.statsDiskFormats { + display: flex; + flex-direction: column; + gap: 6px; +} + +.statsDiskFormatRow { + display: grid; + grid-template-columns: 60px 1fr 80px; + align-items: center; + gap: 10px; + font-size: 12px; +} + +.statsDiskFormatName { + font-weight: 600; + color: rgba(255, 255, 255, 0.8); +} + +.statsDiskFormatBar { + height: 8px; + background: rgba(255, 255, 255, 0.05); + border-radius: 4px; + overflow: hidden; +} + +.statsDiskFormatFill { + height: 100%; + background: linear-gradient(90deg, rgb(var(--accent-rgb)) 0%, rgba(var(--accent-rgb), 0.6) 100%); + border-radius: 4px; +} + +.statsDiskFormatSize { + text-align: right; + color: rgba(255, 255, 255, 0.55); + font-variant-numeric: tabular-nums; +} + +.statsDbStorageWrap { + display: flex; + align-items: center; + gap: 24px; + margin-top: 8px; +} + +.statsDbChartContainer { + position: relative; + width: 180px; + height: 180px; + flex-shrink: 0; + isolation: isolate; +} + +.statsDbTotal { + position: absolute; + inset: 0; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + pointer-events: none; + z-index: 1; +} + +.statsDbTotalValue { + font-size: 20px; + font-weight: 700; + color: rgba(255, 255, 255, 0.85); + text-align: center; +} + +.statsDbTotalLabel { + font-size: 10px; + color: rgba(255, 255, 255, 0.4); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.statsDbLegend { + flex: 1; + display: flex; + flex-direction: column; + gap: 5px; + min-width: 0; +} + +.statsDbLegendItem { + display: flex; + align-items: center; + gap: 8px; + font-size: 12px; + color: rgba(255, 255, 255, 0.6); +} + +.statsDbLegendName { + flex: 1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.statsDbLegendSize { + font-variant-numeric: tabular-nums; + color: rgba(255, 255, 255, 0.4); + font-size: 11px; +} + +.statsEmpty, +.statsLoading { + text-align: center; + padding: 80px 20px; + color: rgba(255, 255, 255, 0.5); +} + +.statsEmptyIcon { + font-size: 48px; + margin-bottom: 16px; +} + +.statsEmpty h3 { + font-size: 1.2rem; + color: rgba(255, 255, 255, 0.7); + margin-bottom: 8px; +} + +.statsEmpty p, +.statsSubtleError, +.emptyListState { + font-size: 0.88rem; + line-height: 1.5; + color: rgba(255, 255, 255, 0.5); +} + +.emptyListState { + padding: 12px; +} + +.statsSubtleError { + padding: 12px 0; +} + +@media (max-width: 768px) { + .statsContainer { + margin: 8px; + padding: 12px; + border-radius: 16px; + } + + .statsHeader { + flex-direction: column; + align-items: flex-start; + padding: 12px; + margin: -12px -12px 20px; + min-height: 0; + gap: 10px; + border-top-left-radius: 16px; + border-top-right-radius: 16px; + } + + .statsHeaderControls { + width: 100%; + flex-direction: column; + align-items: stretch; + } + + .statsTimeRange { + width: 100%; + } + + .statsRangeButton { + padding: 6px 12px; + font-size: 11px; + flex: 1; + text-align: center; + } + + .statsSyncControls { + width: 100%; + justify-content: space-between; + } + + .statsStandaloneNotice { + max-width: none; + text-align: left; + } + + .headerIcon { + width: 100px; + height: 100px; + } + + .statsOverview { + grid-template-columns: repeat(2, 1fr); + gap: 8px; + } + + .statsCard { + padding: 12px 10px; + border-radius: 10px; + } + + .statsCardValue { + font-size: 18px; + } + + .statsCardLabel { + font-size: 9px; + } + + .statsMainGrid { + grid-template-columns: 1fr; + gap: 14px; + } + + .statsLeftCol, + .statsRightCol { + gap: 14px; + } + + .statsSectionCard { + padding: 12px; + } + + .chartContainer { + height: 160px; + } + + .statsGenreChartContainer, + .statsDbStorageWrap { + flex-direction: column; + align-items: center; + } + + .statsHealthGrid { + grid-template-columns: 1fr 1fr; + } +} diff --git a/webui/src/routes/stats/-ui/stats-page.tsx b/webui/src/routes/stats/-ui/stats-page.tsx new file mode 100644 index 00000000..63259a7d --- /dev/null +++ b/webui/src/routes/stats/-ui/stats-page.tsx @@ -0,0 +1,929 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { useNavigate } from '@tanstack/react-router'; +import { type ReactNode, useEffect, useRef, useState } from 'react'; +import { + Bar, + BarChart, + CartesianGrid, + Pie, + PieChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; + +import type { ShellBridge } from '@/platform/shell/bridge'; + +import { useReactPageShell, useShellStatus } from '@/platform/shell/route-controllers'; + +import type { + StatsAlbumRow, + StatsArtistRow, + StatsDbStoragePayload, + StatsHealth, + StatsLibraryDiskUsagePayload, + StatsRange, + StatsRecentTrack, + StatsTrackRow, +} from '../-stats.types'; + +import { + invalidateStatsQueries, + listeningStatsStatusQueryOptions, + resolveStatsTrack, + statsCachedQueryOptions, + statsDbStorageQueryOptions, + statsLibraryDiskUsageQueryOptions, + streamStatsTrack, + triggerListeningStatsSync, +} from '../-stats.api'; +import { + EMPTY_STATS_OVERVIEW, + formatBytes, + formatCompactNumber, + formatDbStorageValue, + formatListeningTime, + formatRelativePlayedAt, + formatTotalDuration, + getStatsRangeLabel, + getTopArtistBubbles, + groupDbStorageTables, + hasStatsData, + STATS_DB_STORAGE_COLORS, + STATS_ENRICHMENT_SERVICES, + STATS_GENRE_COLORS, +} from '../-stats.helpers'; +import { Route } from '../route'; +import styles from './stats-page.module.css'; + +const STATS_TOOLTIP_STYLE = { + background: 'rgba(12, 12, 12, 0.96)', + border: '1px solid rgba(255,255,255,0.08)', + borderRadius: 10, + color: '#fff', +} as const; + +const STATS_TOOLTIP_WRAPPER_STYLE = { + zIndex: 3, +} as const; + +const STATS_CHART_CURSOR = { + fill: 'rgba(var(--accent-rgb), 0.12)', +} as const; + +export function StatsPage() { + const bridge = useReactPageShell('stats'); + + const navigate = useNavigate({ from: Route.fullPath }); + const queryClient = useQueryClient(); + const { range } = Route.useSearch(); + const syncTimeoutRef = useRef(null); + const [syncing, setSyncing] = useState(false); + + const cachedStatsQuery = useQuery({ + ...statsCachedQueryOptions(range), + }); + const listeningStatusQuery = useQuery({ + ...listeningStatsStatusQueryOptions(), + }); + const dbStorageQuery = useQuery({ + ...statsDbStorageQueryOptions(), + }); + const diskUsageQuery = useQuery({ + ...statsLibraryDiskUsageQueryOptions(), + }); + + useEffect(() => { + return () => { + if (syncTimeoutRef.current) { + window.clearTimeout(syncTimeoutRef.current); + } + }; + }, []); + + const syncMutation = useMutation({ + mutationFn: triggerListeningStatsSync, + onMutate: () => { + setSyncing(true); + }, + onSuccess: () => { + window.showToast?.('Syncing listening data...', 'info'); + syncTimeoutRef.current = window.setTimeout(() => { + void invalidateStatsQueries(queryClient); + setSyncing(false); + window.showToast?.('Listening stats updated', 'success'); + }, 5000); + }, + onError: (error) => { + setSyncing(false); + window.showToast?.(error instanceof Error ? error.message : 'Sync failed', 'error'); + }, + }); + + const cachedStats = cachedStatsQuery.data; + const overview = cachedStats?.overview ?? EMPTY_STATS_OVERVIEW; + const hasData = hasStatsData(overview); + const lastSynced = listeningStatusQuery.data?.stats?.last_poll ?? null; + const shellStatus = useShellStatus(); + const isStandalone = shellStatus?.media_server?.type === 'soulsync'; + + const onRangeChange = (nextRange: StatsRange) => { + void navigate({ + to: Route.fullPath, + search: { range: nextRange }, + replace: true, + }); + }; + + const openArtistDetail = (artistId: string | number, artistName: string) => { + bridge.navigateToArtistDetail(artistId, artistName); + }; + + return ( +
+
+
+ Stats +

Listening Stats

+
+
+
+ {(['7d', '30d', '12m', 'all'] as const).map((option) => ( + + ))} +
+
+ {isStandalone ? ( + + Standalone mode: manual sync unavailable + + ) : ( + <> + + {lastSynced ? `Last synced: ${lastSynced}` : 'Not synced yet'} + + + + )} +
+
+
+ + {cachedStatsQuery.isPending ? ( + + ) : cachedStatsQuery.error ? ( + + ) : hasData ? ( + <> + +
+
+ +
+ +
+ + +
+
+ +
+ +
+ + + playStatsTrack(bridge, track)} + /> + +
+
+ + openArtistDetail(artistId, artistName)} + /> + openArtistDetail(artistId, artistName)} + /> + + + openArtistDetail(artistId, artistName)} + /> + + + openArtistDetail(artistId, artistName)} + onPlay={(track) => playStatsTrack(bridge, track)} + /> + +
+
+ + + + + + + + + + + + + + ) : ( + + )} +
+ ); +} + +function OverviewCards({ + overview, +}: { + overview: Partial<{ + total_plays: number; + total_time_ms: number; + unique_artists: number; + unique_albums: number; + unique_tracks: number; + }>; +}) { + const cards = [ + { label: 'Total Plays', value: formatCompactNumber(overview.total_plays) }, + { label: 'Listening Time', value: formatListeningTime(overview.total_time_ms) }, + { label: 'Artists', value: formatCompactNumber(overview.unique_artists) }, + { label: 'Albums', value: formatCompactNumber(overview.unique_albums) }, + { label: 'Tracks', value: formatCompactNumber(overview.unique_tracks) }, + ]; + + return ( +
+ {cards.map((card) => ( +
+
{card.value}
+
{card.label}
+
+ ))} +
+ ); +} + +function StatsSectionCard({ + children, + fullWidth = false, + title, +}: { + children: ReactNode; + fullWidth?: boolean; + title: string; +}) { + return ( +
+
{title}
+ {children} +
+ ); +} + +function StatsActivityChart({ timeline }: { timeline: Array<{ date: string; plays: number }> }) { + return ( + + + + + + + + + + ); +} + +function StatsGenreChart({ + genres, +}: { + genres: Array<{ genre: string; play_count: number; percentage: number }>; +}) { + const topGenres = genres.slice(0, 10).map((genre, index) => ({ + ...genre, + fill: STATS_GENRE_COLORS[index % STATS_GENRE_COLORS.length], + })); + return ( + + + + + + + ); +} + +function StatsGenreLegend({ + genres, +}: { + genres: Array<{ genre: string; play_count: number; percentage: number }>; +}) { + const topGenres = genres.slice(0, 10); + + return ( +
+ {topGenres.map((genre, index) => ( +
+ + {genre.genre} + {genre.percentage}% +
+ ))} +
+ ); +} + +function TopArtistsVisual({ + artists, + onArtistSelect, +}: { + artists: StatsArtistRow[]; + onArtistSelect: (artistId: string | number, artistName: string) => void; +}) { + const topArtists = getTopArtistBubbles(artists); + if (topArtists.length === 0) return null; + + return ( +
+
+ {topArtists.map(({ artist, percent, size }) => { + const isClickable = artist.id !== null && artist.id !== undefined; + return ( + + ); + })} +
+
+ ); +} + +function StatsRankedArtists({ + artists, + onArtistSelect, +}: { + artists: StatsArtistRow[]; + onArtistSelect: (artistId: string | number, artistName: string) => void; +}) { + return ( +
+ {artists.length === 0 ? : null} + {artists.map((artist, index) => ( +
+ {index + 1} + {artist.image_url ? ( + + ) : ( +
+ )} +
+
+ {artist.id ? ( + + ) : ( + artist.name + )} + {artist.soul_id && !String(artist.soul_id).startsWith('soul_unnamed_') ? ( + SoulID + ) : null} +
+
+ {artist.global_listeners + ? `${formatCompactNumber(artist.global_listeners)} global listeners` + : ''} +
+
+ + {formatCompactNumber(artist.play_count)} plays + +
+ ))} +
+ ); +} + +function StatsRankedAlbums({ + albums, + onArtistSelect, +}: { + albums: StatsAlbumRow[]; + onArtistSelect: (artistId: string | number, artistName: string) => void; +}) { + return ( +
+ {albums.length === 0 ? : null} + {albums.map((album, index) => ( +
+ {index + 1} + {album.image_url ? ( + + ) : ( +
+ )} +
+
{album.name}
+
+ {album.artist_id ? ( + + ) : ( + album.artist || '' + )} +
+
+ + {formatCompactNumber(album.play_count)} plays + +
+ ))} +
+ ); +} + +function StatsRankedTracks({ + tracks, + onArtistSelect, + onPlay, +}: { + tracks: StatsTrackRow[]; + onArtistSelect: (artistId: string | number, artistName: string) => void; + onPlay: (track: { title: string; artist: string; album: string }) => Promise; +}) { + return ( +
+ {tracks.length === 0 ? : null} + {tracks.map((track, index) => ( +
+ {index + 1} + {track.image_url ? ( + + ) : ( +
+ )} +
+
{track.name}
+
+ {track.artist_id ? ( + + ) : ( + track.artist || '' + )} + {track.album ? ` · ${track.album}` : ''} +
+
+ + + {formatCompactNumber(track.play_count)} plays + +
+ ))} +
+ ); +} + +function StatsRecentPlays({ + tracks, + onPlay, +}: { + tracks: StatsRecentTrack[]; + onPlay: (track: { title: string; artist: string; album: string }) => Promise; +}) { + return ( +
+ {tracks.length === 0 ? : null} + {tracks.map((track, index) => ( +
+ + {track.title} + {track.artist || ''} + {formatRelativePlayedAt(track.played_at)} +
+ ))} +
+ ); +} + +function StatsLibraryHealth({ health }: { health: StatsHealth }) { + const totalTracks = health.total_tracks ?? 0; + const formatEntries = Object.entries(health.format_breakdown ?? {}); + const formatTotal = formatEntries.reduce((sum, [, count]) => sum + count, 0) || 1; + const formatColors: Record = { + FLAC: '#3b82f6', + MP3: '#f97316', + Opus: '#a855f7', + AAC: '#14b8a6', + OGG: '#eab308', + WAV: '#ec4899', + Other: '#555555', + }; + + return ( +
+
+
+
Format Breakdown
+
+ {formatEntries.map(([format, count]) => { + const percentage = ((count / formatTotal) * 100).toFixed(1); + return ( +
+ {Number(percentage) > 8 ? format : ''} +
+ ); + })} +
+
+
+
+ {formatCompactNumber(health.unplayed_count)} ({health.unplayed_percentage || 0}%) +
+
Unplayed Tracks
+
+
+
+ {formatTotalDuration(health.total_duration_ms)} +
+
Total Duration
+
+
+
{formatCompactNumber(totalTracks)}
+
Total Tracks
+
+
+
+ {STATS_ENRICHMENT_SERVICES.map((service) => { + const percent = health.enrichment_coverage?.[service.key] || 0; + return ( +
+ {service.label} +
+
+
+ {percent}% +
+ ); + })} +
+
+ ); +} + +function StatsDiskUsage({ + error, + payload, +}: { + error: unknown; + payload: StatsLibraryDiskUsagePayload | undefined; +}) { + if (error) { + return ; + } + + const hasData = payload?.has_data && !!payload.total_bytes; + const formats = Object.entries(payload?.by_format ?? {}).sort((a, b) => b[1] - a[1]); + const max = formats[0]?.[1] || 1; + const tracksWithSize = payload?.tracks_with_size || 0; + const tracksWithoutSize = payload?.tracks_without_size || 0; + + return ( +
+
+
+ {hasData ? formatBytes(payload?.total_bytes) : '—'} +
+
+ {hasData + ? `${tracksWithSize.toLocaleString()} tracks measured${ + tracksWithoutSize > 0 + ? ` (+${tracksWithoutSize.toLocaleString()} pending next Deep Scan)` + : '' + }` + : tracksWithoutSize > 0 + ? `Run a Deep Scan to populate (${tracksWithoutSize.toLocaleString()} tracks pending)` + : 'No tracks in library yet'} +
+
+
+ {formats.map(([format, bytes]) => { + const width = Math.max(2, Math.round((bytes / max) * 100)); + return ( +
+ {format.toUpperCase()} +
+
+
+ {formatBytes(bytes)} +
+ ); + })} +
+
+ ); +} + +function StatsDbStorage({ + error, + payload, +}: { + error: unknown; + payload: StatsDbStoragePayload | undefined; +}) { + if (error) { + return ; + } + + const tables = groupDbStorageTables(payload?.tables ?? []).map((table, index) => ({ + ...table, + fill: STATS_DB_STORAGE_COLORS[index % STATS_DB_STORAGE_COLORS.length], + })); + const method = payload?.method; + + return ( +
+
+ + + + + + +
+
+ {formatDbStorageValue(payload?.total_file_size || 0, method)} +
+
Total Size
+
+
+
+ {tables.map((table, index) => ( +
+ + {table.name} + + {formatDbStorageValue(table.size, method)} + +
+ ))} +
+
+ ); +} + +function StatsEmptyState() { + return ( +
+
📊
+

No Listening Data Yet

+

+ Enable "Listening Stats" in Settings to start tracking your listening activity + from your media server. +

+
+ ); +} + +function SectionLoadingState() { + return
Loading listening stats...
; +} + +function SectionErrorState({ message }: { message: string }) { + return ( +
+

Failed to load listening stats

+

{message}

+
+ ); +} + +function SectionSubtleError({ message }: { message: string }) { + return
{message}
; +} + +function EmptyListState({ message }: { message: string }) { + return
{message}
; +} + +async function playStatsTrack( + bridge: ShellBridge, + track: { title: string; artist: string; album: string }, +) { + try { + const resolvedTrack = await resolveStatsTrack(track.title, track.artist); + if (resolvedTrack) { + await bridge.playLibraryTrack( + { + id: resolvedTrack.id, + title: resolvedTrack.title, + file_path: resolvedTrack.file_path, + bitrate: resolvedTrack.bitrate, + artist_id: resolvedTrack.artist_id, + album_id: resolvedTrack.album_id, + _stats_image: resolvedTrack.image_url || null, + }, + resolvedTrack.album_title || track.album, + resolvedTrack.artist_name || track.artist, + ); + return; + } + } catch { + // Library resolve is best-effort; fall through to stream lookup on failure. + } + + bridge.showLoadingOverlay(`Searching for ${track.title}...`); + try { + const streamResult = await streamStatsTrack(track.title, track.artist, track.album); + bridge.hideLoadingOverlay(); + + if (streamResult) { + await bridge.startStream(streamResult); + return; + } + + window.showToast?.('Track not found in library or any source', 'error'); + } catch (error) { + bridge.hideLoadingOverlay(); + window.showToast?.(getErrorMessage(error), 'error'); + } +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : 'Unknown error'; +} diff --git a/webui/src/routes/stats/route.tsx b/webui/src/routes/stats/route.tsx new file mode 100644 index 00000000..358d6f36 --- /dev/null +++ b/webui/src/routes/stats/route.tsx @@ -0,0 +1,33 @@ +import { createFileRoute, redirect } from '@tanstack/react-router'; + +import { getProfileHomePath } from '@/platform/shell/bridge'; + +import { listeningStatsStatusQueryOptions, statsCachedQueryOptions } from './-stats.api'; +import { statsSearchSchema } from './-stats.types'; +import { StatsPage } from './-ui/stats-page'; + +export const Route = createFileRoute('/stats')({ + validateSearch: statsSearchSchema, + beforeLoad: ({ context }) => { + const { bridge } = context.shell; + + if (!bridge.isPageAllowed('stats')) { + throw redirect({ href: getProfileHomePath(bridge), replace: true }); + } + }, + loaderDeps: ({ search }) => ({ + range: search.range, + }), + loader: async ({ context, deps }) => { + await Promise.all([ + context.queryClient.ensureQueryData(statsCachedQueryOptions(deps.range)), + context.queryClient + .fetchQuery({ + ...listeningStatsStatusQueryOptions(), + retry: false, + }) + .catch(() => undefined), + ]); + }, + component: StatsPage, +}); diff --git a/webui/src/test/shell-bridge.ts b/webui/src/test/shell-bridge.ts new file mode 100644 index 00000000..0c0cbe4b --- /dev/null +++ b/webui/src/test/shell-bridge.ts @@ -0,0 +1,24 @@ +import { vi } from 'vitest'; + +import type { ShellBridge, ShellPageId } from '@/platform/shell/bridge'; + +export function createShellBridge(overrides: Partial = {}): ShellBridge { + const bridge: ShellBridge = { + getCurrentProfileContext: vi.fn(() => ({ profileId: 2, isAdmin: true })), + isPageAllowed: vi.fn(() => true), + getProfileHomePage: vi.fn<() => ShellPageId>(() => 'discover'), + resolveLegacyPath: vi.fn<(pathname: string) => ShellPageId | null>(() => 'search'), + setActivePageChrome: vi.fn(), + activateLegacyPath: vi.fn(), + cancelSimilarArtistsLoad: vi.fn(), + navigateToArtistDetail: vi.fn(), + playLibraryTrack: vi.fn(), + showReactHost: vi.fn(), + startStream: vi.fn(), + showLoadingOverlay: vi.fn(), + hideLoadingOverlay: vi.fn(), + }; + + Object.assign(bridge, overrides); + return bridge; +} diff --git a/webui/static/amazon.svg b/webui/static/amazon.svg new file mode 100644 index 00000000..a7ddd600 --- /dev/null +++ b/webui/static/amazon.svg @@ -0,0 +1,5 @@ + + a + + + diff --git a/webui/static/api-monitor.js b/webui/static/api-monitor.js index 01e27f0a..d53d0667 100644 --- a/webui/static/api-monitor.js +++ b/webui/static/api-monitor.js @@ -4,17 +4,19 @@ const _rateMonitorState = {}; const _RATE_GAUGE_SERVICES = [ 'spotify', 'itunes', 'deezer', 'lastfm', 'genius', - 'musicbrainz', 'audiodb', 'tidal', 'qobuz', 'discogs', + 'musicbrainz', 'audiodb', 'tidal', 'qobuz', 'discogs', 'amazon', ]; const _RATE_GAUGE_LABELS = { spotify: 'Spotify', itunes: 'Apple Music', deezer: 'Deezer', lastfm: 'Last.fm', genius: 'Genius', musicbrainz: 'MusicBrainz', audiodb: 'AudioDB', tidal: 'Tidal', qobuz: 'Qobuz', discogs: 'Discogs', + amazon: 'Amazon Music', }; const _RATE_GAUGE_COLORS = { spotify: '#1DB954', itunes: '#FC3C44', deezer: '#A238FF', lastfm: '#D51007', genius: '#FFFF64', musicbrainz: '#BA478F', audiodb: '#00BCD4', tidal: '#00FFFF', qobuz: '#FF6B35', discogs: '#D4A574', + amazon: '#FF9900', }; // SVG constants — 240° arc, gap at bottom @@ -614,6 +616,26 @@ async function fetchAndUpdateActivityFeed() { // Cache last feed signature to avoid unnecessary DOM rebuilds (prevents blink) let _lastActivityFeedSig = ''; +// Activity items carry `timestamp` (Unix epoch seconds) — `activity.time` +// is a human label like "Now" that doesn't parse as a date. Use the +// epoch for relative-time formatting; fall back to the label only +// when no timestamp is present (legacy items, future shapes). +function _activityTimeAgo(activity) { + const ts = activity && activity.timestamp; + if (typeof ts !== 'number' || !isFinite(ts)) { + return (activity && activity.time) || ''; + } + const diffMs = Date.now() - ts * 1000; + if (diffMs < 60000) return 'Just now'; + const mins = Math.floor(diffMs / 60000); + if (mins < 60) return `${mins}m ago`; + const hours = Math.floor(mins / 60); + if (hours < 24) return `${hours}h ago`; + const days = Math.floor(hours / 24); + if (days < 30) return `${days}d ago`; + return `${Math.floor(days / 30)}mo ago`; +} + function updateActivityFeed(activities) { const feedContainer = document.getElementById('dashboard-activity-feed'); if (!feedContainer) return; @@ -644,7 +666,7 @@ function updateActivityFeed(activities) { // Just update timestamps without rebuilding DOM const timeEls = feedContainer.querySelectorAll('.activity-time'); items.forEach((activity, i) => { - if (timeEls[i]) timeEls[i].textContent = timeAgo(activity.time); + if (timeEls[i]) timeEls[i].textContent = _activityTimeAgo(activity); }); return; } @@ -660,7 +682,7 @@ function updateActivityFeed(activities) {

${escapeHtml(activity.title)}

${escapeHtml(activity.subtitle)}

-

${timeAgo(activity.time)}

+

${_activityTimeAgo(activity)}

`; feedContainer.appendChild(activityElement); @@ -952,7 +974,9 @@ async function initializeWatchlistPage() { if (artist.itunes_artist_id) sourceBadges.push('iTunes'); if (artist.deezer_artist_id) sourceBadges.push('Deezer'); if (artist.discogs_artist_id) sourceBadges.push('Discogs'); - const artistPrimaryId = artist.spotify_artist_id || artist.itunes_artist_id || artist.deezer_artist_id || artist.discogs_artist_id; + if (artist.musicbrainz_artist_id) sourceBadges.push('MusicBrainz'); + if (artist.amazon_artist_id) sourceBadges.push('Amazon'); + const artistPrimaryId = artist.spotify_artist_id || artist.itunes_artist_id || artist.deezer_artist_id || artist.discogs_artist_id || artist.musicbrainz_artist_id || artist.amazon_artist_id; return `
iTunes'); if (artist.deezer_artist_id) sourceBadges.push('Deezer'); if (artist.discogs_artist_id) sourceBadges.push('Discogs'); - const artistPrimaryId = artist.spotify_artist_id || artist.itunes_artist_id || artist.deezer_artist_id || artist.discogs_artist_id; + if (artist.musicbrainz_artist_id) sourceBadges.push('MusicBrainz'); + if (artist.amazon_artist_id) sourceBadges.push('Amazon'); + const artistPrimaryId = artist.spotify_artist_id || artist.itunes_artist_id || artist.deezer_artist_id || artist.discogs_artist_id || artist.musicbrainz_artist_id || artist.amazon_artist_id; return `
🌐Default (${globalLabel}) @@ -2258,7 +2287,7 @@ async function openWatchlistArtistDetailView(artistId, artistName) { return; } - const { config, artist, recent_releases, spotify_artist_id, itunes_artist_id, deezer_artist_id, discogs_artist_id } = data; + const { config, artist, recent_releases, spotify_artist_id, itunes_artist_id, deezer_artist_id, discogs_artist_id, musicbrainz_artist_id } = data; // Remove existing overlay if any const existing = document.querySelector('.watchlist-artist-detail-overlay'); @@ -2287,6 +2316,25 @@ async function openWatchlistArtistDetailView(artistId, artistName) { if (artist.mood) metaTags.push(`${escapeHtml(artist.mood)}`); if (artist.label) metaTags.push(`${escapeHtml(artist.label)}`); + let discogId = null; + let discogSource = null; + const activeSrc = (currentMusicSourceName || '').toLowerCase(); + if (activeSrc.includes('spotify') && spotify_artist_id) { + discogId = spotify_artist_id; discogSource = 'spotify'; + } else if (activeSrc.includes('discogs') && discogs_artist_id) { + discogId = discogs_artist_id; discogSource = 'discogs'; + } else if (activeSrc.includes('deezer') && deezer_artist_id) { + discogId = deezer_artist_id; discogSource = 'deezer'; + } else if (activeSrc.includes('musicbrainz') && musicbrainz_artist_id) { + discogId = musicbrainz_artist_id; discogSource = 'musicbrainz'; + } else if (itunes_artist_id) { + discogId = itunes_artist_id; discogSource = 'itunes'; + } else { + discogId = spotify_artist_id || discogs_artist_id || deezer_artist_id || musicbrainz_artist_id || itunes_artist_id; + discogSource = spotify_artist_id ? 'spotify' : discogs_artist_id ? 'discogs' : deezer_artist_id ? 'deezer' : musicbrainz_artist_id ? 'musicbrainz' : 'itunes'; + } + const discogHref = discogId ? buildArtistDetailPath(discogId, discogSource) : '#'; + overlay.innerHTML = ` ${artist.banner_url ? `
@@ -2369,7 +2417,7 @@ async function openWatchlistArtistDetailView(artistId, artistName) {
- + View Discography
@@ -2381,28 +2429,6 @@ async function openWatchlistArtistDetailView(artistId, artistName) { closeWatchlistArtistDetailView(); }); - overlay.querySelector('.watchlist-detail-discog-action').addEventListener('click', () => { - // Use the ID matching the active metadata source - let discogId, source; - const activeSrc = (currentMusicSourceName || '').toLowerCase(); - if (activeSrc.includes('spotify') && spotify_artist_id) { - discogId = spotify_artist_id; source = 'spotify'; - } else if (activeSrc.includes('discogs') && discogs_artist_id) { - discogId = discogs_artist_id; source = 'discogs'; - } else if (activeSrc.includes('deezer') && deezer_artist_id) { - discogId = deezer_artist_id; source = 'deezer'; - } else if (itunes_artist_id) { - discogId = itunes_artist_id; source = 'itunes'; - } else { - discogId = spotify_artist_id || discogs_artist_id || deezer_artist_id || itunes_artist_id; - source = spotify_artist_id ? 'spotify' : discogs_artist_id ? 'discogs' : deezer_artist_id ? 'deezer' : 'itunes'; - } - if (discogId) { - closeWatchlistArtistDetailView(); - navigateToArtistDetail(discogId, artistName, source); - } - }); - overlay.querySelector('.watchlist-detail-settings-action').addEventListener('click', () => { // Remove overlay immediately so it doesn't block the config modal const detailOverlay = document.querySelector('.watchlist-artist-detail-overlay'); diff --git a/webui/static/core.js b/webui/static/core.js index 13285531..9a1bd1e4 100644 --- a/webui/static/core.js +++ b/webui/static/core.js @@ -67,6 +67,11 @@ let deezerPlaylistStates = {}; let deezerArlPlaylists = []; let deezerArlPlaylistsLoaded = false; +// --- Qobuz Playlist State Management (mirrors Tidal — github issue #677) --- +let qobuzPlaylists = []; +let qobuzPlaylistStates = {}; // Key: playlist_id, Value: playlist state with phases +let qobuzPlaylistsLoaded = false; + // --- Beatport Chart State Management (Similar to YouTube/Tidal) --- let beatportChartStates = {}; // Key: chart_hash, Value: chart state with phases let beatportContentState = { @@ -201,6 +206,13 @@ let artistsSearchController = null; let artistCompletionController = null; // Track ongoing completion check to cancel when navigating away let similarArtistsController = null; // Track ongoing similar artists stream to cancel when navigating away +function cancelSimilarArtistsLoad() { + if (similarArtistsController) { + similarArtistsController.abort(); + similarArtistsController = null; + } +} + // --- Lazy Background Image Observer --- // Watches elements with data-bg-src, applies background-image when visible, unobserves after. const lazyBgObserver = new IntersectionObserver((entries) => { @@ -446,6 +458,7 @@ function initializeWebSocket() { socket.on('enrichment:genius-enrichment', (data) => updateGeniusEnrichmentStatusFromData(data)); socket.on('enrichment:tidal-enrichment', (data) => updateTidalEnrichmentStatusFromData(data)); socket.on('enrichment:qobuz-enrichment', (data) => updateQobuzEnrichmentStatusFromData(data)); + socket.on('enrichment:amazon-enrichment', (data) => updateAmazonEnrichmentStatusFromData(data)); socket.on('enrichment:hydrabase', (data) => updateHydrabaseStatusFromData(data)); socket.on('enrichment:repair', (data) => updateRepairStatusFromData(data)); socket.on('enrichment:soulid', (data) => updateSoulIDStatusFromData(data)); @@ -464,7 +477,13 @@ function initializeWebSocket() { // Phase 5 event listeners (sync/discovery progress + scans) socket.on('sync:progress', (data) => updateSyncProgressFromData(data)); socket.on('discovery:progress', (data) => updateDiscoveryProgressFromData(data)); - socket.on('scan:watchlist', (data) => updateWatchlistScanFromData(data)); + socket.on('scan:watchlist', (data) => { + updateWatchlistScanFromData(data); + const watchlistBtn = document.querySelector('.nav-button[data-page="watchlist"]'); + if (watchlistBtn) { + watchlistBtn.classList.toggle('nav-watchlist-scanning', data.status === 'scanning'); + } + }); socket.on('scan:media', (data) => updateMediaScanFromData(data)); socket.on('wishlist:stats', (data) => updateWishlistStatsFromData(data)); // Phase 6: Automation progress @@ -501,6 +520,7 @@ function handleServiceStatusUpdate(data) { const isSoulsyncStandalone = data.media_server?.type === 'soulsync'; _isSoulsyncStandalone = isSoulsyncStandalone; document.querySelectorAll('.sync-to-server-btn, [id$="-sync-btn"], [onclick*="startPlaylistSync"], [onclick*="syncPlaylistToServer"], [onclick*="startDecadeSync"]').forEach(btn => { + if (btn.id === 'stats-sync-btn') return; // React stats page owns this control now. if (isSoulsyncStandalone) { btn.dataset.hiddenByStandalone = '1'; btn.style.display = 'none'; @@ -675,6 +695,7 @@ const GENIUS_LOGO_URL = 'https://images.genius.com/8ed669cadd956443e29c70361ec4f const TIDAL_LOGO_URL = 'https://www.svgrepo.com/show/519734/tidal.svg'; const QOBUZ_LOGO_URL = 'https://www.svgrepo.com/show/504778/qobuz.svg'; const DISCOGS_LOGO_URL = 'https://www.svgrepo.com/show/305957/discogs.svg'; +const AMAZON_LOGO_URL = '/static/amazon.svg'; function getAudioDBLogoURL() { const el = document.querySelector('img.audiodb-logo'); return el ? el.src : null; } // --- Wishlist Modal Persistence State Management --- diff --git a/webui/static/discover.js b/webui/static/discover.js index 3142a22b..b5303e5d 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -269,10 +269,9 @@ function displayDiscoverHeroArtist(artist) { if (discographyBtn && artistId) { discographyBtn.setAttribute('data-artist-id', artistId); discographyBtn.setAttribute('data-artist-name', artist.artist_name); - // Source the click handler will pass to navigateToArtistDetail. Without - // this, source-only hero artists (which is the typical case — they - // come from discover similar-artists, not the library) get looked up - // as library IDs and 404. Backend always includes artist.source. + discographyBtn.href = buildArtistDetailPath(artistId, artist.source || null); + // Keep the source on the link so source-only hero artists resolve to + // the correct artist-detail URL instead of being treated as library IDs. if (artist.source) discographyBtn.setAttribute('data-source', artist.source); else discographyBtn.removeAttribute('data-source'); // Also store both IDs for cross-source operations @@ -387,6 +386,7 @@ async function watchAllHeroArtists(btn) { // Cache for recommended artists data so reopening is instant let _recommendedArtistsCache = null; +let _recommendedArtistsSource = null; async function openRecommendedArtistsModal() { let modal = document.getElementById('recommended-artists-modal'); @@ -404,7 +404,7 @@ async function openRecommendedArtistsModal() { // If cached, render instantly and refresh watchlist statuses if (_recommendedArtistsCache) { modal.style.display = 'flex'; - renderRecommendedArtistsModal(modal, _recommendedArtistsCache); + renderRecommendedArtistsModal(modal, _recommendedArtistsCache, _recommendedArtistsSource); checkRecommendedWatchlistStatuses(_recommendedArtistsCache); return; } @@ -444,13 +444,14 @@ async function openRecommendedArtistsModal() { return; } - // Render cards immediately with fallback images - _recommendedArtistsCache = data.artists; - renderRecommendedArtistsModal(modal, data.artists); - // Phase 2: Enrich with images/genres progressively in batches of 50 // Skip artists that already have cached metadata from the initial response const source = data.source || 'spotify'; + // Render cards immediately with fallback images + _recommendedArtistsCache = data.artists; + _recommendedArtistsSource = source; + renderRecommendedArtistsModal(modal, data.artists, source); + const idKey = source === 'spotify' ? 'spotify_artist_id' : source === 'deezer' ? 'deezer_artist_id' : 'itunes_artist_id'; const allIds = data.artists .filter(a => !a.image_url) // Only enrich artists without cached images @@ -521,7 +522,7 @@ async function openRecommendedArtistsModal() { } } -function renderRecommendedArtistsModal(modal, artists) { +function renderRecommendedArtistsModal(modal, artists, source = null) { modal.innerHTML = `
@@ -555,30 +556,36 @@ function renderRecommendedArtistsModal(modal, artists) { const similarText = artist.occurrence_count > 1 ? `Similar to ${artist.occurrence_count} in your watchlist` : 'Similar to an artist in your watchlist'; + const artistSource = artist.source || source || _recommendedArtistsSource || ''; return `
+ data-artist-id="${artist.artist_id}" + data-artist-source="${escapeHtml(artistSource)}"> -
- ${artist.image_url ? ` - ${escapeHtml(artist.artist_name)} - ` : ` -
🎤
- `} -
-
- ${escapeHtml(artist.artist_name)} - ${similarText} -
${genreTags}
-
+ +
+ ${artist.image_url ? ` + ${escapeHtml(artist.artist_name)} + ` : ` +
🎤
+ `} +
+
+ ${escapeHtml(artist.artist_name)} + ${similarText} +
${genreTags}
+
+
`; }).join('')} @@ -595,15 +602,6 @@ function renderRecommendedArtistsModal(modal, artists) { if (watchlistBtn) { e.stopPropagation(); toggleRecommendedWatchlist(watchlistBtn); - return; - } - - const card = e.target.closest('.recommended-artist-card'); - if (card) { - const artistId = card.getAttribute('data-artist-id'); - const nameEl = card.querySelector('.recommended-card-name'); - const artistName = nameEl ? nameEl.textContent : ''; - viewRecommendedArtistDiscography(artistId, artistName); } }); } @@ -734,11 +732,6 @@ async function checkRecommendedWatchlistStatuses(artists) { } } -async function viewRecommendedArtistDiscography(artistId, artistName) { - closeRecommendedArtistsModal(); - navigateToArtistDetail(artistId, artistName); -} - async function checkAllHeroWatchlistStatus() { const btn = document.getElementById('discover-hero-watch-all'); if (!btn || !discoverHeroArtists || discoverHeroArtists.length === 0) return; @@ -806,26 +799,6 @@ function jumpToDiscoverHeroSlide(index) { updateDiscoverHeroIndicators(); } -async function viewDiscoverHeroDiscography() { - const button = document.getElementById('discover-hero-discography'); - if (!button) return; - - const artistId = button.getAttribute('data-artist-id'); - const artistName = button.getAttribute('data-artist-name'); - // Pass the source so /api/artist-detail knows to synthesize from that - // metadata provider instead of doing a local DB lookup. Hero similar - // artists are almost always source-only (not in the library). - const source = button.getAttribute('data-source') || null; - - if (!artistId || !artistName) { - console.error('No artist data found for discography view'); - return; - } - - console.log(`🎵 Navigating to artist detail for: ${artistName} (source: ${source || 'library'})`); - navigateToArtistDetail(artistId, artistName, source); -} - function showDiscoverHeroEmpty() { const titleEl = document.getElementById('discover-hero-title'); const subtitleEl = document.getElementById('discover-hero-subtitle'); @@ -3940,6 +3913,35 @@ function hideSeasonalSections() { } } +function _buildDiscoverArtistContext(source, artistName, sourceData = {}, albumData = {}) { + const normalizedSource = (source || '').toString().toLowerCase(); + const albumArtist = Array.isArray(albumData.artists) ? albumData.artists[0] : null; + const activeSource = (source || sourceData.active_source || sourceData.source || '').toString().toLowerCase(); + const context = { + ...sourceData, + id: sourceData.active_source_id || sourceData.artist_id || albumArtist?.id || '', + name: artistName || sourceData.artist_name || sourceData.name || albumArtist?.name || '', + source: normalizedSource || activeSource || '', + spotify_artist_id: sourceData.spotify_artist_id || sourceData.artist_spotify_id || ((normalizedSource || activeSource) === 'spotify' ? albumArtist?.id : '') || '', + itunes_artist_id: sourceData.itunes_artist_id || sourceData.artist_itunes_id || ((normalizedSource || activeSource) === 'itunes' ? albumArtist?.id : '') || '', + deezer_artist_id: sourceData.deezer_artist_id || sourceData.artist_deezer_id || sourceData.deezer_id || ((normalizedSource || activeSource) === 'deezer' ? albumArtist?.id : '') || '', + discogs_artist_id: sourceData.discogs_artist_id || sourceData.artist_discogs_id || sourceData.discogs_id || '', + amazon_artist_id: sourceData.amazon_artist_id || sourceData.artist_amazon_id || sourceData.amazon_id || '', + soul_id: sourceData.soul_id || sourceData.hydrabase_artist_id || '', + }; + + const sourceIdBySource = { + spotify: context.spotify_artist_id, + itunes: context.itunes_artist_id, + deezer: context.deezer_artist_id, + discogs: context.discogs_artist_id, + amazon: context.amazon_artist_id, + hydrabase: context.soul_id, + }; + context.id = sourceIdBySource[normalizedSource || activeSource] || context.id; + return context; +} + async function openDownloadModalForSeasonalAlbum(albumIndex) { const album = discoverSeasonalAlbums[albumIndex]; if (!album) { @@ -3999,14 +4001,12 @@ async function openDownloadModalForSeasonalAlbum(albumIndex) { const virtualPlaylistId = `seasonal_album_${albumId}`; // Pass proper artist/album context for album download (1 worker + source reuse) - const artistContext = { - name: album.artist_name, - source: source - }; + const artistContext = _buildDiscoverArtistContext(source, album.artist_name, album, albumData); const albumContext = { id: albumData.id, name: albumData.name, + source: source, album_type: albumData.album_type || 'album', total_tracks: albumData.total_tracks || 0, release_date: albumData.release_date || '', @@ -4354,6 +4354,31 @@ async function unblockDiscoveryArtist(id, name) { let _yourArtistsCtrl = null; +function _pickArtistDetailSource(artist) { + if (!artist) return { id: '', source: '' }; + const sourceFields = { + spotify: 'spotify_artist_id', + deezer: 'deezer_artist_id', + itunes: 'itunes_artist_id', + discogs: 'discogs_artist_id', + amazon: 'amazon_artist_id', + musicbrainz: 'musicbrainz_artist_id', + hydrabase: 'soul_id', + }; + const active = String(artist.active_source || artist.source || '').toLowerCase(); + const activeField = sourceFields[active]; + if (activeField && artist[activeField]) { + return { id: String(artist[activeField]), source: active }; + } + for (const [source, field] of Object.entries(sourceFields)) { + if (artist[field]) return { id: String(artist[field]), source }; + } + if (artist.active_source_id && activeField) { + return { id: String(artist.active_source_id), source: active }; + } + return { id: '', source: '' }; +} + async function loadYourArtists() { if (!_yourArtistsCtrl) { _yourArtistsCtrl = createDiscoverSectionController({ @@ -4454,12 +4479,10 @@ function _renderYourArtistCard(artist) { ).join(''); const watchlistClass = artist.on_watchlist ? 'active' : ''; - const hasId = artist.active_source_id && artist.active_source_id !== ''; + const detailSource = _pickArtistDetailSource(artist); + const hasId = detailSource.id && detailSource.id !== ''; - // Navigate to Artists page (name click) — source artist id, needs inline view - const navAction = hasId - ? `event.stopPropagation(); navigateToArtistDetail('${escapeForInlineJs(artist.active_source_id)}', '${escapeForInlineJs(artist.artist_name)}')` - : ''; + const detailHref = hasId ? buildArtistDetailPath(detailSource.id, detailSource.source) : ''; // Open info modal (card body click) — pass pool ID so we can look up all data const infoAction = hasId @@ -4487,7 +4510,9 @@ function _renderYourArtistCard(artist) {
${originDots}
-
${_esc(artist.artist_name)}
+ ${hasId + ? `${_esc(artist.artist_name)}` + : `
${_esc(artist.artist_name)}
`}
`; @@ -4637,10 +4662,10 @@ async function openYourArtistInfoModal(poolId) { Explore - + `; } } catch (err) { @@ -6537,9 +6562,9 @@ function _artMapSetupInteraction(canvas) {
ⓘ Artist Info
-
+ 💿 View Discography -
+
👁 ${node.type === 'watchlist' ? 'On Watchlist' : 'Add to Watchlist'}
@@ -6724,11 +6749,12 @@ async function openYourArtistInfoModal_direct(node) { let bestId = '', bestSource = ''; // Check what the active source is const activeSource = window._yaActiveSource || 'spotify'; - const sourceOrder = activeSource === 'spotify' ? ['spotify_id', 'itunes_id', 'deezer_id', 'discogs_id'] - : activeSource === 'itunes' ? ['itunes_id', 'spotify_id', 'deezer_id', 'discogs_id'] - : activeSource === 'deezer' ? ['deezer_id', 'spotify_id', 'itunes_id', 'discogs_id'] - : ['spotify_id', 'itunes_id', 'deezer_id', 'discogs_id']; - const sourceMap = { spotify_id: 'spotify', itunes_id: 'itunes', deezer_id: 'deezer', discogs_id: 'discogs' }; + const sourceOrder = activeSource === 'spotify' ? ['spotify_id', 'itunes_id', 'deezer_id', 'discogs_id', 'musicbrainz_id'] + : activeSource === 'itunes' ? ['itunes_id', 'spotify_id', 'deezer_id', 'discogs_id', 'musicbrainz_id'] + : activeSource === 'deezer' ? ['deezer_id', 'spotify_id', 'itunes_id', 'discogs_id', 'musicbrainz_id'] + : activeSource === 'musicbrainz' ? ['musicbrainz_id', 'spotify_id', 'itunes_id', 'deezer_id', 'discogs_id'] + : ['spotify_id', 'itunes_id', 'deezer_id', 'discogs_id', 'musicbrainz_id']; + const sourceMap = { spotify_id: 'spotify', itunes_id: 'itunes', deezer_id: 'deezer', discogs_id: 'discogs', musicbrainz_id: 'musicbrainz' }; for (const key of sourceOrder) { if (node[key]) { bestId = node[key]; bestSource = sourceMap[key]; break; } } @@ -6815,7 +6841,7 @@ function _renderByltTrackCard(t) { return `
- ${t.image_url ? `` : '
🎵
'} + ${t.image_url ? `` : '
🎵
'}
${_esc(t.name)}
${_esc(t.artist)}
@@ -6955,7 +6981,7 @@ async function openCacheDiscoverAlbum(sectionKey, index) { duration_ms: track.duration_ms || 0, track_number: track.track_number || 0, }; }); - const artistContext = { id: albumData.artists?.[0]?.id || '', name: artistName, source: resolvedSource }; + const artistContext = _buildDiscoverArtistContext(resolvedSource, artistName, item, albumData); const albumContext = { id: albumData.id, name: albumData.name, album_type: albumData.album_type || 'album', total_tracks: albumData.total_tracks || 0, release_date: albumData.release_date || '', images: albumData.images || [] }; await openDownloadMissingModalForYouTube(`discover_cache_${resolvedId}`, albumData.name, spotifyTracks, artistContext, albumContext); hideLoadingOverlay(); @@ -7015,11 +7041,7 @@ async function openCacheDiscoverAlbum(sectionKey, index) { }; }); - const artistContext = { - id: albumData.artists?.[0]?.id || '', - name: item.artist_name || albumData.artists?.[0]?.name || '', - source: source, - }; + const artistContext = _buildDiscoverArtistContext(source, item.artist_name || albumData.artists?.[0]?.name || '', item, albumData); const albumContext = { id: albumData.id, name: albumData.name, @@ -7220,9 +7242,9 @@ async function openGenreDeepDive(genre) { // Always open on Artists page with discography — pass source for correct routing const imgUrl = _esc(a.image_url || ''); const artSource = _esc(a.source || ''); - const clickAction = `onclick="document.getElementById('genre-deep-dive-modal').remove();navigateToArtistDetail('${_esc(a.entity_id)}','${_esc(a.name)}','${artSource}' || null)"`; + const detailHref = a.entity_id ? buildArtistDetailPath(a.entity_id, a.source || null) : '#'; const srcClass = (a.source || '').toLowerCase(); - return `
+ return `
${!a.image_url ? '🎤' : ''}
@@ -7230,7 +7252,7 @@ async function openGenreDeepDive(genre) {
${_esc(a.name)}
${a.followers ? `
${_fmtNum(a.followers)} followers
` : ''} ${a.library_id ? '
In Library
' : ''} -
`; + `; }).join('')}
`; @@ -7939,11 +7961,7 @@ async function openDownloadModalForRecentAlbum(albumIndex) { const virtualPlaylistId = `discover_album_${albumId}`; // CRITICAL FIX: Pass proper artist/album context for modal display - const artistContext = { - id: source === 'spotify' ? album.artist_spotify_id : source === 'deezer' ? album.artist_deezer_id : album.artist_itunes_id, - name: album.artist_name, - source: source - }; + const artistContext = _buildDiscoverArtistContext(source, album.artist_name, album, albumData); const albumContext = { id: albumData.id, @@ -8717,4 +8735,3 @@ if (document.readyState === 'loading') { } // ============================================================================ - diff --git a/webui/static/downloads.js b/webui/static/downloads.js index 29c58f96..18798508 100644 --- a/webui/static/downloads.js +++ b/webui/static/downloads.js @@ -64,10 +64,11 @@ function _wingItAction(urlHash, action) { const tracks = state.tracks || state.rawTracks || state.playlist?.tracks || []; const name = state.playlistName || state.name || state.playlist?.name || 'Playlist'; const isTidal = state.is_tidal_playlist; + const isQobuz = state.is_qobuz_playlist; const isLB = state.is_listenbrainz_playlist; const isBeatport = state.is_beatport_playlist; const isDeezer = state.is_deezer_playlist; - const source = isLB ? 'ListenBrainz' : isTidal ? 'Tidal' : isDeezer ? 'Deezer' : isBeatport ? 'Beatport' : 'YouTube'; + const source = isLB ? 'ListenBrainz' : isTidal ? 'Tidal' : isQobuz ? 'Qobuz' : isDeezer ? 'Deezer' : isBeatport ? 'Beatport' : 'YouTube'; if (!tracks.length) { showToast('No tracks available for Wing It', 'error'); @@ -347,10 +348,11 @@ async function _wingItFromModal(urlHash) { const tracks = state.tracks || state.rawTracks || state.playlist?.tracks || []; const name = state.playlistName || state.name || state.playlist?.name || 'Playlist'; const isTidal = state.is_tidal_playlist; + const isQobuz = state.is_qobuz_playlist; const isLB = state.is_listenbrainz_playlist; const isBeatport = state.is_beatport_playlist; const isDeezer = state.is_deezer_playlist; - const source = isLB ? 'ListenBrainz' : isTidal ? 'Tidal' : isDeezer ? 'Deezer' : isBeatport ? 'Beatport' : 'YouTube'; + const source = isLB ? 'ListenBrainz' : isTidal ? 'Tidal' : isQobuz ? 'Qobuz' : isDeezer ? 'Deezer' : isBeatport ? 'Beatport' : 'YouTube'; if (!tracks.length) { showToast('No tracks available for Wing It', 'error'); @@ -501,7 +503,10 @@ async function openDownloadMissingModalForYouTube(virtualPlaylistId, playlistNam const heroContext = isDiscoverAlbum && album && artist ? { type: 'album', artist: { + ...artist, name: artist.name, + id: artist.id || artist.artist_id || null, + source: artist.source || album.source || null, image_url: artist.image_url || null }, album: { @@ -579,7 +584,7 @@ async function openDownloadMissingModalForYouTube(virtualPlaylistId, playlistNam onchange="updateTrackSelectionCount('${virtualPlaylistId}')"> ${index + 1} - ${escapeHtml(track.name)} + ${renderModalTrackPlayButton(virtualPlaylistId, index)}${escapeHtml(track.name)} ${escapeHtml(formatArtists(track.artists))} ${formatDuration(track.duration_ms)} 🔍 Pending @@ -630,13 +635,6 @@ async function openDownloadMissingModalForYouTube(virtualPlaylistId, playlistNam hideLoadingOverlay(); } -function _navigateToArtistFromModal(artistId, artistName, imageUrl, source, playlistId) { - if (!artistName) return; - // Close the download modal - if (playlistId) closeDownloadMissingModal(playlistId); - navigateToArtistDetail(artistId || artistName, artistName, source || null); -} - async function closeDownloadMissingModal(playlistId) { const process = activeDownloadProcesses[playlistId]; if (!process) { @@ -1508,46 +1506,54 @@ async function selectWishlistCategory(category) { // Sort tracks by track number albumData.tracks.sort((a, b) => a.trackNumber - b.trackNumber); - const tracksListHTML = albumData.tracks.map(track => ` + const tracksListHTML = albumData.tracks.map(track => { + const safeTrackId = escapeHtml(track.spotifyTrackId || ''); + const safeTrackName = escapeHtml(track.name || 'Unknown Track'); + return `
- ${track.name} -
- `).join(''); + `; + }).join(''); // Handle missing album images with a placeholder + const safeAlbumId = escapeHtml(albumId); + const safeAlbumName = escapeHtml(albumData.albumName || 'Unknown Album'); + const safeArtistName = escapeHtml(albumData.artistName || 'Unknown Artist'); + const safeAlbumImage = escapeHtml(albumData.albumImage || '').replace(/'/g, '''); const albumImageStyle = albumData.albumImage - ? `background-image: url('${albumData.albumImage}')` + ? `background-image: url('${safeAlbumImage}')` : `background: linear-gradient(135deg, rgba(30, 30, 30, 0.9) 0%, rgba(50, 50, 50, 0.9) 100%); display: flex; align-items: center; justify-content: center; font-size: 40px;`; const albumImageContent = albumData.albumImage ? '' : '💿'; albumsHTML += `
-
+
${albumImageContent}
-
${albumData.albumName}
-
${albumData.artistName}
+
${safeAlbumName}
+
${safeArtistName}
${albumData.tracks.length} track${albumData.tracks.length !== 1 ? 's' : ''}
-
▼
- @@ -1599,20 +1605,25 @@ async function selectWishlistCategory(category) { const albumImage = spotifyData?.album?.images?.[0]?.url || ''; const spotifyTrackId = track.spotify_track_id || track.id || ''; + const safeTrackId = escapeHtml(spotifyTrackId); + const safeTrackName = escapeHtml(trackName); + const safeArtistName = escapeHtml(artistName); + const safeAlbumName = escapeHtml(albumName); + const safeAlbumImage = escapeHtml(albumImage).replace(/'/g, '''); tracksHTML += `
-
+
-
${trackName}
-
${artistName} • ${albumName}
+
${safeTrackName}
+
${safeArtistName} • ${safeAlbumName}
-
@@ -2134,7 +2145,7 @@ async function openDownloadMissingWishlistModal(category = null, selectedTrackId ${tracks.map((track, index) => ` ${index + 1} - ${escapeHtml(track.name)} + ${renderModalTrackPlayButton(playlistId, index)}${escapeHtml(track.name)} ${escapeHtml(formatArtists(track.artists))} 🔍 Pending - @@ -2478,6 +2489,7 @@ async function startMissingTracksProcess(playlistId) { const requestBody = { tracks: selectedTracks, force_download_all: forceDownloadAll || isWingIt, + ignore_manual_matches: forceDownloadAll, wing_it: isWingIt, }; @@ -2603,6 +2615,89 @@ function updateTrackAnalysisResults(playlistId, results) { } } +function getModalTrackArtistName(track, fallbackArtist = '') { + const formatted = formatArtists(track?.artists); + if (formatted && formatted !== 'Unknown Artist') return formatted; + return track?.artist_name || track?.artist || fallbackArtist || formatted || ''; +} + +function getModalTrackAlbumTitle(track, process = null) { + if (track?.album) { + if (typeof track.album === 'string') return track.album; + if (track.album.name) return track.album.name; + if (track.album.title) return track.album.title; + } + if (process?.album) { + return process.album.name || process.album.title || ''; + } + return ''; +} + +function renderModalTrackPlayButton(playlistId, trackIndex) { + return ``; +} + +async function playTrackFromLibraryOrStream(track, albumTitle = '', artistName = '') { + const title = track?.title || track?.name || ''; + if (!title) { + showToast('No track title available to play', 'error'); + return; + } + + if (track?.file_path && typeof playLibraryTrack === 'function') { + await playLibraryTrack({ + id: track.id || track.track_id || null, + title, + file_path: track.file_path, + _stats_image: track._stats_image || track.album_thumb_url || null, + bitrate: track.bitrate, + artist_id: track.artist_id, + album_id: track.album_id + }, albumTitle, artistName); + return; + } + + try { + const res = await fetch('/api/stats/resolve-track', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ title, artist: artistName }) + }); + const data = await res.json(); + if (data.success && data.track && data.track.file_path && typeof playLibraryTrack === 'function') { + await playLibraryTrack({ + ...data.track, + title: data.track.title || title, + _stats_image: data.track.album_thumb_url || data.track.artist_thumb_url || null + }, data.track.album_title || albumTitle, data.track.artist_name || artistName); + return; + } + } catch (e) { + console.debug('Library resolve failed before stream fallback:', e); + } + + if (typeof _gsPlayTrack === 'function') { + await _gsPlayTrack(title, artistName, albumTitle); + } else { + showToast('Playback is not available here', 'error'); + } +} + +async function playDownloadModalTrack(playlistId, trackIndex) { + const process = activeDownloadProcesses[playlistId]; + const track = process?.tracks?.[trackIndex] || playlistTrackCache[playlistId]?.[trackIndex]; + if (!track) { + showToast('Track is no longer available in this modal', 'error'); + return; + } + + await playTrackFromLibraryOrStream( + track, + getModalTrackAlbumTitle(track, process), + getModalTrackArtistName(track, process?.artist?.name || '') + ); +} + // ============================================================================ @@ -3247,6 +3342,41 @@ async function downloadCandidate(taskId, candidate, trackName) { } } +async function approveQuarantineFromDownloadRow(button) { + const entryId = button?.dataset?.entryId || ''; + if (!entryId) { + showToast('Open Quarantine to approve this file.', 'warning'); + return; + } + + const confirmed = await showConfirmDialog({ + title: 'Approve Quarantined File', + message: 'Import this quarantined file and skip quarantine checks for this approved pass?', + confirmText: 'Approve & Import', + cancelText: 'Cancel', + }); + if (!confirmed) return; + + const originalText = button.textContent; + button.disabled = true; + button.textContent = 'Approving...'; + try { + const response = await fetch(`/api/quarantine/${encodeURIComponent(entryId)}/approve`, { method: 'POST' }); + const data = await response.json(); + if (data.success) { + showToast('Approved quarantined file. Re-running post-processing.', 'success'); + } else { + showToast(`Approve failed: ${data.error || 'Unknown error'}`, 'error'); + button.disabled = false; + button.textContent = originalText; + } + } catch (error) { + showToast(`Approve failed: ${error.message}`, 'error'); + button.disabled = false; + button.textContent = originalText; + } +} + function closeCandidatesModal() { const overlay = document.getElementById('candidates-modal-overlay'); if (overlay) { @@ -3255,6 +3385,74 @@ function closeCandidatesModal() { } } +function _downloadModalBundleProgressPercent(bundle) { + if (!bundle) return 0; + const raw = bundle.progress_percent ?? bundle.progress ?? 0; + let progress = Number(raw); + if (!Number.isFinite(progress)) progress = 0; + if (progress <= 1) progress *= 100; + return Math.max(0, Math.min(100, Math.round(progress))); +} + +function _downloadModalFormatBytes(bytes) { + const value = Number(bytes); + if (!Number.isFinite(value) || value <= 0) return ''; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let size = value; + let unit = 0; + while (size >= 1024 && unit < units.length - 1) { + size /= 1024; + unit += 1; + } + const decimals = size >= 10 || unit === 0 ? 0 : 1; + return `${size.toFixed(decimals)} ${units[unit]}`; +} + +function _downloadModalFormatSpeed(bytesPerSecond) { + const formatted = _downloadModalFormatBytes(bytesPerSecond); + return formatted ? `${formatted}/s` : ''; +} + +function _downloadModalSourceLabel(source) { + const labels = { + torrent: 'Torrent', + usenet: 'Usenet', + soulseek: 'Soulseek', + youtube: 'YouTube', + tidal: 'Tidal', + qobuz: 'Qobuz', + hifi: 'HiFi', + deezer_dl: 'Deezer', + amazon: 'Amazon', + lidarr: 'Lidarr', + soundcloud: 'SoundCloud' + }; + const key = String(source || '').toLowerCase(); + return labels[key] || (source ? String(source) : 'Release'); +} + +function _downloadModalBundleStateLabel(state) { + const labels = { + searching: 'searching for release', + downloading: 'downloading release', + staged: 'matching tracks', + failed: 'release failed' + }; + const key = String(state || '').toLowerCase(); + return labels[key] || (state ? String(state).replace(/_/g, ' ') : 'downloading release'); +} + +function _downloadModalBundleProgressText(bundle) { + const percent = _downloadModalBundleProgressPercent(bundle); + const source = _downloadModalSourceLabel(bundle && bundle.source); + const state = _downloadModalBundleStateLabel(bundle && bundle.state); + const release = bundle && bundle.release ? ` - ${bundle.release}` : ''; + const speed = _downloadModalFormatSpeed(bundle && bundle.speed); + const size = _downloadModalFormatBytes(bundle && bundle.size); + const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; + return `${source} ${state} ${percent}%${release}${detail}`; +} + function processModalStatusUpdate(playlistId, data) { // This function contains ALL the existing polling logic from startModalDownloadPolling // Extracted so it can be called from both individual and batched polling @@ -3303,6 +3501,33 @@ function processModalStatusUpdate(playlistId, data) { // Auto-save M3U file for playlists after analysis autoSavePlaylistM3U(playlistId); } + } else if (data.phase === 'album_downloading') { + const analysisFill = document.getElementById(`analysis-progress-fill-${playlistId}`); + const analysisText = document.getElementById(`analysis-progress-text-${playlistId}`); + if (analysisFill) analysisFill.style.width = '100%'; + if (analysisText) analysisText.textContent = 'Analysis complete!'; + + const bundle = data.album_bundle || {}; + const percent = _downloadModalBundleProgressPercent(bundle); + const downloadFill = document.getElementById(`download-progress-fill-${playlistId}`); + const downloadText = document.getElementById(`download-progress-text-${playlistId}`); + if (downloadFill) downloadFill.style.width = `${percent}%`; + if (downloadText) { + downloadText.textContent = _downloadModalBundleProgressText(bundle); + downloadText.title = 'SoulSync downloads one album release first, then matches the selected tracks from the staged files.'; + } + + const modal = document.getElementById(`download-missing-modal-${playlistId}`); + if (modal) { + modal.querySelectorAll('[id^="download-"]').forEach(statusEl => { + if (!statusEl.id.startsWith(`download-${playlistId}-`)) return; + if (!statusEl.textContent || statusEl.textContent === '-' || statusEl.textContent.includes('Pending')) { + statusEl.textContent = 'Waiting for release'; + statusEl.classList.add('album-bundle-waiting'); + statusEl.title = 'The album release is downloading first. Tracks will move to processing once SoulSync can match files from it.'; + } + }); + } } else if (data.phase === 'downloading' || data.phase === 'complete' || data.phase === 'error') { console.debug(`📊 [Status Update] Processing ${data.phase} phase for playlistId: ${playlistId}, tasks: ${(data.tasks || []).length}`); @@ -3371,6 +3596,7 @@ function processModalStatusUpdate(playlistId, data) { const actionsEl = document.getElementById(`actions-${playlistId}-${task.track_index}`); let statusText = ''; + let isQuarantinedTask = false; // V2 SYSTEM: Handle UI state override for cancelling tasks if (isV2Task && uiState === 'cancelling' && task.status !== 'cancelled') { statusText = '🔄 Cancelling...'; @@ -3382,7 +3608,19 @@ function processModalStatusUpdate(playlistId, data) { case 'post_processing': statusText = '⌛ Processing...'; break; case 'completed': statusText = '✅ Completed'; completedCount++; break; case 'not_found': statusText = '🔇 Not Found'; notFoundCount++; break; - case 'failed': statusText = '❌ Failed'; failedOrCancelledCount++; break; + case 'failed': { + // Distinguish quarantine outcomes from generic + // failures — the file is recoverable, not lost. + const _em = (task.error_message || '').toLowerCase(); + if (_em.includes('integrity check failed') || _em.includes('bit depth filter') || _em.includes('verification failed') || _em.includes('quarantin')) { + isQuarantinedTask = true; + statusText = '🛡️ Quarantined'; + } else { + statusText = '❌ Failed'; + } + failedOrCancelledCount++; + break; + } case 'cancelled': statusText = '🚫 Cancelled'; failedOrCancelledCount++; break; default: statusText = `⚪ ${task.status}`; break; } @@ -3423,6 +3661,13 @@ function processModalStatusUpdate(playlistId, data) { const onclickHandler = isV2Task ? 'cancelTrackDownloadV2' : 'cancelTrackDownload'; actionsEl.innerHTML = ``; } + } else if (actionsEl && task.status === 'failed' && isQuarantinedTask && task.quarantine_entry_id) { + const entryId = escapeHtml(task.quarantine_entry_id); + actionsEl.innerHTML = ``; + const approveBtn = actionsEl.querySelector('.approve-quarantine-inline-btn'); + if (approveBtn) { + approveBtn.addEventListener('click', () => approveQuarantineFromDownloadRow(approveBtn)); + } } else if (actionsEl && ['completed', 'failed', 'cancelled', 'not_found', 'post_processing'].includes(task.status)) { actionsEl.innerHTML = '-'; // No actions available for terminal or processing states } @@ -3521,12 +3766,9 @@ function processModalStatusUpdate(playlistId, data) { setTimeout(() => loadServerPlaylists(), 2000); } - // Auto-close wishlist modal when completed (for auto-processing) + // Keep visible wishlist results open so failed tracks can be reviewed. if (playlistId === 'wishlist') { - console.log('🔄 [Auto-Wishlist] Auto-closing completed wishlist modal to enable next cycle'); - setTimeout(() => { - closeDownloadMissingModal(playlistId); - }, 3000); // 3-second delay to show completion message + console.log('[Wishlist] Leaving completed wishlist modal open for failed-track review'); } // Check if any other processes still need polling @@ -5594,13 +5836,13 @@ function _gsRenderFromState(state) { if (dbArtists.length) { h += '
📚 In Your Library
'; - h += dbArtists.map(a => `
${a.image_url ? `` : '🎤'}
${_escToast(a.name)}
Library
`).join(''); + h += dbArtists.map(a => `${a.image_url ? `
` : '
🎤
'}
${_escToast(a.name)}
Library
`).join(''); h += '
'; } if (artists.length) { h += `
🎤 Artists ${srcLabel}
`; - h += artists.map(a => `
${a.image_url ? `` : '🎤'}
${_escToast(a.name)}
`).join(''); + h += artists.map(a => `${a.image_url ? `
` : '
🎤
'}
${_escToast(a.name)}
`).join(''); h += '
'; } @@ -5674,13 +5916,6 @@ async function _gsLazyLoadArtistImages() { } } -function _gsClickArtist(id, name, isLibrary) { - _gsDeactivate(); - const activeSource = _gsController && _gsController.state.activeSource; - const source = isLibrary ? null : (activeSource || null); - navigateToArtistDetail(id, name, source); -} - async function _gsClickAlbum(albumId, albumName, artistName, imageUrl, source) { _gsDeactivate(); // Same flow as handleEnhancedSearchAlbumClick — fetch album, open download modal @@ -6296,4 +6531,3 @@ const additionalStyles = ` document.head.insertAdjacentHTML('beforeend', additionalStyles); // ============================================================================ - diff --git a/webui/static/enrichment.js b/webui/static/enrichment.js index 23c44935..0cf9ced6 100644 --- a/webui/static/enrichment.js +++ b/webui/static/enrichment.js @@ -1205,6 +1205,112 @@ if (document.readyState === 'loading') { } } +// =================================================================== +// AMAZON MUSIC ENRICHMENT WORKER +// =================================================================== + +async function updateAmazonEnrichmentStatus() { + if (socketConnected) return; + if (document.hidden) return; + try { + const response = await fetch('/api/enrichment/amazon/status'); + if (!response.ok) { console.warn('Amazon enrichment status endpoint unavailable'); return; } + const data = await response.json(); + updateAmazonEnrichmentStatusFromData(data); + } catch (error) { + console.error('Error updating Amazon enrichment status:', error); + } +} + +function updateAmazonEnrichmentStatusFromData(data) { + const button = document.getElementById('amazon-enrich-button'); + if (!button) return; + + button.classList.remove('active', 'paused', 'complete'); + if (data.paused) { + button.classList.add('paused'); + } else if (data.idle) { + button.classList.add('complete'); + } else if (data.running && !data.paused) { + button.classList.add('active'); + } + + const tooltipStatus = document.getElementById('amazon-enrich-tooltip-status'); + const tooltipCurrent = document.getElementById('amazon-enrich-tooltip-current'); + const tooltipProgress = document.getElementById('amazon-enrich-tooltip-progress'); + + if (tooltipStatus) { + if (data.paused) { tooltipStatus.textContent = data.yield_reason === 'downloads' ? 'Yielding for downloads' : 'Paused'; } + else if (data.idle) { tooltipStatus.textContent = 'Complete'; } + else if (data.running) { tooltipStatus.textContent = 'Running'; } + else { tooltipStatus.textContent = 'Idle'; } + } + + if (tooltipCurrent) { + if (data.idle) { + tooltipCurrent.textContent = 'All items processed'; + } else if (data.current_item && data.current_item.name) { + tooltipCurrent.textContent = `Now: ${data.current_item.name}`; + } else { + tooltipCurrent.textContent = 'No active matches'; + } + } + + if (data.progress && tooltipProgress) { + const artists = data.progress.artists || {}; + const albums = data.progress.albums || {}; + const tracks = data.progress.tracks || {}; + const currentType = data.current_item?.type; + let progressText = ''; + const artistsComplete = artists.matched >= artists.total; + const albumsComplete = albums.matched >= albums.total; + if (currentType === 'artist' || (!artistsComplete && !currentType)) { + progressText = `Artists: ${artists.matched || 0} / ${artists.total || 0} (${artists.percent || 0}%)`; + } else if (currentType === 'album' || (!albumsComplete && !currentType)) { + progressText = `Albums: ${albums.matched || 0} / ${albums.total || 0} (${albums.percent || 0}%)`; + } else { + progressText = `Tracks: ${tracks.matched || 0} / ${tracks.total || 0} (${tracks.percent || 0}%)`; + } + tooltipProgress.textContent = progressText; + } +} + +async function toggleAmazonEnrichment() { + try { + const button = document.getElementById('amazon-enrich-button'); + if (!button) return; + const isRunning = button.classList.contains('active'); + const endpoint = isRunning ? '/api/enrichment/amazon/pause' : '/api/enrichment/amazon/resume'; + const response = await fetch(endpoint, { method: 'POST' }); + if (!response.ok) { + throw new Error(`Failed to ${isRunning ? 'pause' : 'resume'} Amazon enrichment`); + } + await updateAmazonEnrichmentStatus(); + console.log(`Amazon enrichment ${isRunning ? 'paused' : 'resumed'}`); + } catch (error) { + console.error('Error toggling Amazon enrichment:', error); + showToast(`Error: ${error.message}`, 'error'); + } +} + +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => { + const button = document.getElementById('amazon-enrich-button'); + if (button) { + button.addEventListener('click', toggleAmazonEnrichment); + updateAmazonEnrichmentStatus(); + setInterval(updateAmazonEnrichmentStatus, 2000); + } + }); +} else { + const button = document.getElementById('amazon-enrich-button'); + if (button) { + button.addEventListener('click', toggleAmazonEnrichment); + updateAmazonEnrichmentStatus(); + setInterval(updateAmazonEnrichmentStatus, 2000); + } +} + // =================================================================== // HYDRABASE P2P MIRROR WORKER // =================================================================== diff --git a/webui/static/helper.js b/webui/static/helper.js index 736220dc..c29f9009 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -1632,7 +1632,7 @@ const HELPER_CONTENT = { // ─── STATS PAGE ────────────────────────────────────────────────── - '.stats-container': { + '#stats-container': { title: 'Listening Stats', description: 'Analytics dashboard showing your listening activity, top artists/albums/tracks, genre breakdown, library health, and storage usage. Data syncs from your media server.', }, @@ -3413,391 +3413,62 @@ function closeHelperSearch() { // projects that span multiple commits before shipping. Strip the flag at // release time and add a real `date:` line at the top of the version block. const WHATS_NEW = { - '2.5.2': [ - // --- May 13, 2026 — 2.5.2 release --- - { date: 'May 13, 2026 — 2.5.2 release' }, - { title: 'Download Missing Modal: Tracklist Got A Polish Pass', desc: 'visual tune-up only — column layout untouched. hairline row dividers, accent gradient + edge bar on hover, monospace track numbers (glow accent on row hover), monospace tabular duration. status text in both library-match + download-status columns picks up a leading colored dot with a soft halo (green found / amber missing / blue checking / orange downloading / red failed) and pulses while in-flight. artist column centered. soft scrollbar.', page: 'downloads' }, - { title: 'Search Source Picker: Fix Default Always Sticking To Spotify', desc: 'enhanced search + global search source picker always defaulted to spotify even when the user\'s primary metadata source was deezer / itunes / discogs / etc. trace: `shared-helpers.js:createSearchController` reads `/status.metadata_source` to pick the initial active icon, then checks `SOURCE_LABELS[src]` to validate. backend was returning `metadata_source` as a dict (`{source, connected, response_time, ...}` — used elsewhere for connection-state display), so `SOURCE_LABELS[]` was always undefined, the `if` guard never fired, and `state.activeSource` silently stayed at the hardcoded `\'spotify\'` default. fix: read `.source` off the dict (with forward-compat fallback to plain-string in case any older /status response shape predates the dict change). other consumers (core.js sidebar tile, helper.js status checker, search.js display) already used `?.source` correctly — this was the only stale call site.', page: 'search' }, - { title: 'Download Discography: No Longer Caps Prolific Artists At 50 Releases', desc: 'discord report: clicking "download discography" on an artist with a deep catalogue (bach, beatles complete box, dance / electronic artists with hundreds of remixes) only showed ~50 albums in the modal. trace: `MetadataLookupOptions(limit=50, max_pages=0)` was hardcoded at the discography endpoint and the artist-detail discography view. spotify\'s `max_pages=0` already paginates through everything (per-page is clamped to 10 internally) so spotify-primary users were unaffected. but deezer / itunes / discogs / hydrabase all honor the outer `limit` as a hard cap. fix: bump `limit` from 50 to 200 at all three call sites (`web_server.py` discography endpoint + artist-detail view + `core/artist_source_detail.py`). 200 matches iTunes\'s and Discogs\'s own internal caps and covers near-everyone\'s full catalogue. spotify behavior unchanged.', page: 'library' }, - { title: 'Artist Page: "Write Artist Image" Button (Real Artist Photos For Navidrome)', desc: 'github issue #572 (rhwc): navidrome shows album-art-derived thumbnails as artist photos because navidrome has no api for setting an artist image — it only reads `artist.jpg` from the artist folder during library scans. soulsync\'s `update_artist_poster` for navidrome was a no-op. new button on the artist detail page header writes `artist.jpg` to the artist\'s folder on disk: looks up any album track, resolves it through the path resolver (handles docker mount translation like #558 settled on), goes up one level to the artist folder, fetches the artist photo from the configured metadata source priority chain (spotify primary, fallback to deezer / discogs / etc), downloads with content-type validation + atomic write via `.tmp + os.replace`. when active server is navidrome, triggers a library scan immediately so the new file gets indexed. respects existing `artist.jpg` files (asks before overwriting) so user-supplied photos aren\'t clobbered. works for plex / jellyfin too as a fallback layer — both servers also read `artist.jpg` from disk. 26 tests pin the pure helpers in `core/library/artist_image.py`: folder derivation (trailing slash / backslash / empty / non-string), image url picking (missing attr / whitespace strip / non-string), download (non-image content-type / 404 / timeout / empty body), and write (atomic replace / temp-cleanup-on-failure / overwrite guard / missing folder).', page: 'library' }, - { title: 'Library History: Per-Download Audit Trail Modal', desc: 'each download row in library history now has an "audit" button that opens a second modal visualizing the download lifecycle as a vertical chain of decision blocks: request → source selected → source match → verification → post processing → final placement. each step has a status (complete / partial / unknown / error) with a color-coded node, plus a card showing what was decided and the supporting metadata. post-processing step infers observable changes from source-vs-final state (format conversion, file rename via tag template, title/artist rewrite, folder template). new "embedded tags" section below the flow reads the audio file live via mutagen at audit-open time and surfaces every tag actually on the file — title / artist / album / album artist / date / genre / track # / disc # / bpm / mood / style / copyright / publisher / release type+status+country / barcode / catalog # / asin / isrc / replaygain values / cover-art status / lyrics / every source id (spotify, tidal, deezer, musicbrainz, audiodb, lastfm, genius, itunes, beatport ...). file is the single source of truth — a persisted snapshot would drift the moment a background enrichment worker writes more tags. clean fallback when file is missing or unreadable. 19 tests pin the pure mutagen reader: id3 path (TIT2/TPE1/TALB + TXXX user-defined frames + USLT + APIC cover-art), vorbis path (FLAC dict-style + pass-through for unknown _id / _url keys), mp4 stub, format+bitrate+duration metadata, defensive paths (empty path, missing file, mutagen returns None, mutagen raises), stringify edge cases (list / tuple / int / frame-with-text / whitespace). files: core/library/file_tags.py (new mutagen reader), web_server.py (new GET /api/library/history//file-tags endpoint), webui/index.html (audit-overlay modal), webui/static/wishlist-tools.js (renderer + async fetch + tag-grid render), webui/static/style.css (flow + tags section + lyrics block styles).', page: 'wishlist' }, - { title: '$albumtype Folder Template Now Splits EPs / Singles For Non-Spotify Sources', desc: 'discord report (cal): downloading an artist\'s discography with `$albumtype` in the path template put every release under `Album/` regardless of actual type — eps, singles, all dumped into the album folder. trace: the legacy duck-typed album-info builder at `core/metadata/album_tracks.py:_build_album_info_legacy` only checked the `album_type` key. spotify uses `album_type` (lowercase) so spotify discographies worked. but deezer\'s api uses `record_type`, tidal uses `type` (uppercase ALBUM/EP/SINGLE), and some flattened musicbrainz shapes use `primary-type` — none of those matched, all defaulted to `album`. fix: widen the legacy lookup to check `album_type` / `record_type` / `type` / `primary-type` and route the value through a new pure `_normalize_album_type` helper that lowercases + validates against the canonical token set (`album` / `single` / `ep` / `compilation`) and falls back to `album` for unknowns. typed-converter path for spotify / deezer / itunes / discogs / musicbrainz / hydrabase / qobuz unchanged — they were already correct. tidal users were the main offender (no typed converter for dict-shaped tidal data). 25 new tests pin: case-insensitive normalization for each canonical type, compilation preserved (spotify supports it), unknown values default to album, defensive against none / empty / non-string inputs, multi-key precedence (`album_type` wins over `record_type`), each known source shape produces correct token, generic `type=track` / `type=artist` collision case defaults to album rather than poisoning the path.', page: 'tools' }, + '2.6.0': [ + { date: 'May 24, 2026 — 2.6.0 release' }, + { title: 'Qobuz playlist sync', desc: 'new Qobuz tab on the Sync page. Connect Qobuz in Settings → Connections, hit Refresh on the tab, and your Qobuz playlists + Favorite Tracks show up alongside Tidal and Deezer. clicks run the same discovery → sync → download flow as the other sources.', page: 'sync' }, + { title: 'Import search: show when results came from the fallback source', desc: 'if you picked MusicBrainz (or Discogs / iTunes / etc.) as your primary metadata source but the Import album search ended up serving Deezer cards, you had no idea — the chain silently fell through when the primary returned nothing. now each card shows a small "via Deezer" label when the source differs from your primary, and a banner above the grid spells it out when all results came from the fallback. backend behavior unchanged.' }, ], - '2.5.1': [ - // --- May 12, 2026 — 2.5.1 release --- - { date: 'May 12, 2026 — 2.5.1 release' }, - { title: 'Soulseek: Min Delay Between Searches (Fixes ISP Anti-Abuse Trips)', desc: 'reddit report (yelomelo95, bell canada): isp anti-abuse cuts the wan after a burst of slskd searches. soulsync\'s sliding-window cap (35 searches per 220s) prevented soulseek-side bans but allowed all 35 in rapid succession — which is exactly the connection-burst pattern that trips isp throttling. new knob on settings → connections → soulseek: minimum delay between searches (default 0 = disabled, preserves prior behavior). set it to 5-10 seconds if your isp throttles peer-connection spikes. throttle math lifted to a pure `compute_search_wait_seconds` helper so the gate logic is testable independent of asyncio.sleep + the singleton client. 15 new tests pin: defaults / no-throttle, sliding-window cap (legacy), min-delay (the new burst-smoother), max-of-both gates, defensive paths.', page: 'tools' }, - { title: 'Help & Docs: Copy Debug Info Now Reports The Right Music Source + Lists All Services', desc: 'the music_source field always rendered as "unknown" because the code read `_status_cache.get(\'spotify\', {})` — but the cache only has \'media_server\' and \'soulseek\' keys, so the lookup always fell through. same silent miss for spotify_connected and spotify_rate_limited. fix routes those reads through the canonical accessors: `get_primary_source()` for music source (which already accounts for the spotify→deezer auth fallback), `get_spotify_status()` for connection + rate-limit state. also added hydrabase_connected (was missing entirely), youtube_available (always true — yt-dlp + url-based, no auth), hifi_instance_count (separate from connection because each instance is its own endpoint with its own auth), and an always_available_metadata_sources list (deezer / itunes / musicbrainz — public apis, no auth) so the dump reflects the full metadata surface. while in there: removed a local `from core.metadata.status import get_spotify_status` re-import that was making python 3.12 treat the name as a function-scoped local, breaking the new lambda above it (NameError on free variable). 11 new tests at the endpoint boundary pin music_source, spotify_*, hydrabase_*, youtube_available, always_available_metadata_sources, hifi_instance_count, and the defensive paths when each lookup raises.', page: 'tools' }, - { title: 'Download Discography: Skips Tracks Already In Your Library', desc: 'discord report (skowl): clicking download discography on the same artist twice re-queued every track instead of skipping the half already on disk. trace: the endpoint added each track via `add_to_wishlist`, which dedups against the wishlist itself but never checks the library — once a downloaded track leaves the wishlist the next click re-inserts it. fix: same library-ownership check the discography backfill repair job already runs (`db.check_track_exists` at confidence ≥ 0.7). format-agnostic — name + artist + album, no extension comparison — so blasphemy mode (flac → mp3 with original deleted) doesn\'t false-miss. exception during the check returns "not owned" so a transient db hiccup doesn\'t silently nuke the discography fetch (a redundant wishlist add is cheap, a missed track isn\'t). per-album response carries a new `tracks_skipped_owned` counter alongside the artist / content / wishlist skips. 10 new tests at the helper boundary.', page: 'discover' }, - { title: 'Download Discography: No More Cross-Artist Tracks Or Unwanted Remixes', desc: 'issue #559: download discography pulled in tracks from compilations / appears-on albums where the artist was only featured on one or two tracks — every other track on those albums got added too. also ignored your watchlist "include remixes / live / acoustic / instrumental" settings, so one-off discography downloads kept stuffing your wishlist with remix ladders. fix: per-track filter at the endpoint. drops tracks where the requested artist isn\'t named in the track\'s artists list (keeps features, drops unrelated compilation entries). honors `watchlist.global_include_*` settings the same way the discography backfill repair job already does. per-album response carries new skip counts so the ui can show how much got filtered. 21 new tests at the helper boundary.', page: 'discover' }, - { title: 'Album Completeness: "Could Not Determine Album Folder" Error Now Tells You What To Fix', desc: 'github issue #558 (gabistek, navidrome on docker / arch host): clicking auto-fill or fix selected on the album completeness findings page returned a flat "could not determine album folder from existing tracks" error with no diagnostic. trace: the path resolver in `core/library/path_resolver.py` probes transfer + download + `library.music_paths` config + plex api library locations to map db-recorded paths to actual files on disk. for plex users the api auto-discovers the mount paths (per #476). navidrome\'s subsonic api doesn\'t expose filesystem paths at all (only folder names via `getMusicFolders`), and navidrome\'s native rest api on top of that doesn\'t expose them either — there is no api signal we can probe. so for navidrome users in docker, if the path navidrome reports (`/music/artist/album/track.flac`) doesn\'t exist as-is in the soulsync container view AND the user hasn\'t manually configured settings → library → music paths, the resolver returns none and the fix workflow bailed silently. fix: lifted the resolver into a diagnostic-aware variant (`resolve_library_file_path_with_diagnostic` returning a `(resolved, ResolveAttempt)` tuple) that records what was tried — raw-path-existed, base-dirs-probed, whether config_manager / plex_client were wired up. repair_worker uses the diagnostic to render a multi-part error: names the active media server, shows one sample db-recorded path the album\'s tracks have, lists every base directory the resolver actually probed, and points at settings → library → music paths as the actionable fix. user can now read the error and know exactly what to mount or configure. no auto-probing of common docker conventions — too speculative, could resolve to wrong dirs on the suffix-walk if conventional paths happen to contain a partial collision. backwards compatible: legacy `resolve_library_file_path` kept as a thin wrapper that drops the attempt, every existing call site unchanged. 12 new tests pin: tuple shape, raw-path short-circuit attempt fields, base-dirs listed even on walk failure, had-flags reflect caller inputs, error renders active server name + sample path + base dirs, distinguishes empty-base-dirs vs tried-and-failed cases, settings hint always present, defensive against none attempt + missing sample + missing config_manager.', page: 'tools' }, - { title: 'Import History: Clear History Button Now Clears Stuck "Processing" Rows', desc: 'noticed on the import page: clear history left zombie rows behind that all showed "⧗ processing" status from 2-9 days ago. trace: `_record_in_progress` inserts a `status=\'processing\'` row up-front so the ui can render the in-flight import while it runs, then `_finalize_result` updates it to `completed`/`failed` when the import finishes. when the server is restarted mid-import (or the worker crashes), the row never gets finalized — stays at `processing` forever. the clear-history endpoint\'s sql `DELETE ... WHERE status IN (\'completed\', \'approved\', \'failed\', \'needs_identification\', \'rejected\')` didn\'t include `processing`, so those zombies survived every click. fix: add `processing` to the delete list, but guard against nuking actually-live imports by intersecting against `_snapshot_active()` — any folder hash currently registered in the worker\'s in-memory `_active_imports` map is excluded from the delete. `pending_review` deliberately left out so user still has to approve/reject those explicitly. one endpoint touched (`/api/auto-import/clear-completed` in web_server.py). no worker changes. zombie-row pile gets swept on next click, new imports still record + update normally.', page: 'import' }, - { title: 'Auto-Import: Falls Through To Other Metadata Sources When Primary Has No Match', desc: 'discord report (mushy): 16 bandcamp indie albums sat in staging because auto-import couldn\'t identify them. manual search at the bottom of the import music tab found the same albums fine — they just weren\'t on the user\'s primary metadata source (spotify) but existed on tidal/deezer. trace: `_search_metadata_source` in `core/auto_import_worker.py` only queried `get_primary_source()` — single source, no fallback. meanwhile `search_import_albums` (the manual search bar at the bottom of the tab) already iterated the full `get_source_priority(get_primary_source())` chain and broke on first source with results. asymmetric behavior — manual search worked, auto-import didn\'t, same album. fix: lift auto-import to use the same source-chain pattern. try primary first; if it returns nothing OR scores below the 0.4 threshold, fall through to next source in priority order. first source that produces a strong-enough match wins. result dict carries the `source` that actually matched (not the primary name), so downstream `_match_tracks` calls the right client to fetch the album\'s tracklist. defensive per-source try/except so a rate-limited or auth-failed source doesn\'t abort the chain. unconfigured sources (client=None) silently skipped. scoring math lifted to pure helper `_score_album_search_result` so weight tweaks (album 50% / artist 20% / track-count 30%) are pinned at the function boundary independent of the orchestrator. weight constants exposed at module level (`_ALBUM_NAME_WEIGHT`, `_ARTIST_NAME_WEIGHT`, `_TRACK_COUNT_WEIGHT`) — greppable, bumpable in one place. 9 integration tests + 18 scoring-helper tests. integration tests pin: primary-success path unchanged (no fallback fires, only primary client called), primary-empty falls through to next source, primary-weak-score falls through, first fallback success stops the chain (no wasted api calls on remaining sources), all-sources-fail returns None, per-source exception contained, unconfigured-source skipped gracefully, result `source` field reflects winning source, `identification_confidence` from winning source. backwards compatible — single-source users see no change (chain just has one entry).', page: 'import' }, - { title: 'Multi-Artist Tag Settings Now Actually Work (artist_separator + feat_in_title + write_multi_artist)', desc: 'three settings on settings → metadata → tags were partially or completely unimplemented. (1) `write_multi_artist` only worked because of a never-populated `_artists_list` field — `core/metadata/source.py` built `metadata["artist"]` as a hardcoded ", "-joined string but never assigned `metadata["_artists_list"]`, so `core/metadata/enrichment.py:114` always saw an empty list and silently no-op\'d the multi-value tag write. (2) `artist_separator` (default ", ") was referenced in the UI + settings.js save path but ZERO python code read the value — every multi-artist track ended up with hardcoded ", " regardless of what the user picked. (3) `feat_in_title` (when true: pull featured artists into the title as " (feat. X, Y)" and leave only primary in the ARTIST tag — picard convention) had no implementation at all. fix in source.py: populate `_artists_list` from the search response\'s artists array, then build the ARTIST string per the user\'s settings — primary-only when feat_in_title is on (with featured names appended to title; double-append guarded for source titles that already include "feat."), else joined with the configured separator. fix in enrichment.py id3 path: writing TPE1 twice (single-string then list) was overwriting the configured separator. now keeps TPE1 as the display string and writes a separate `TXXX:Artists` frame for the multi-value list (picard convention). vorbis path was already correct (separate "artist" + "artists" keys). deezer-specific upgrade path: deezer\'s `/search` endpoint only returns the primary artist — full contributors live on `/track/`. when source==deezer AND the search response had a single artist AND a track_id is available, enrichment now fetches the per-track endpoint and upgrades the artists list before tag-write. one extra API call per affected deezer track (skipped when search already returned multiple). spotify, tidal, itunes search responses already include all artists so they\'re unaffected. 29 new tests pin: `_artists_list` populated for multi/single/no-artist cases, separator drives ARTIST string (default + custom), single-artist case unaffected by either setting, feat_in_title pulls featured to title + leaves primary in ARTIST, feat_in_title no-op for single artist, double-append guard recognizes 9 source-title variants ("(feat. X)", "(Feat. X)", "(FEAT X)", "(feat X)", "(Featuring X)", "[feat. X]", "ft. X", "(ft X)", "FT. X"), word-boundary regex doesn\'t false-match substrings ("Aftermath" still gets the append), combined-settings precedence (feat_in_title wins over separator for ARTIST string but `_artists_list` carries everyone for the multi-value tag), deezer upgrade fires only when search returned single artist + track_id available, no upgrade for non-deezer sources, upgrade failure falls through to search-result list, no false-positive when /track/ confirms single artist.', page: 'settings' }, - { title: 'AudioDB Enrichment: Track Worker No Longer Stuck In Infinite Retry Loop', desc: 'github issue #553: audiodb track enrichment "stuck" — constant requests, no progress, only error log was a 10s read-timeout from `lookup_track_by_id` repeating against the same track. trace: when an entity already has `audiodb_id` populated (from manual match or earlier scan) but `audiodb_match_status` is NULL, the worker tries a direct ID lookup. if it fails (returns None on timeout — audiodb\'s `track.php` endpoint is slow, 10s timeouts common), the prior code logged "preserving manual match" and returned WITHOUT marking status. row stayed NULL → queue picked it up next tick → tried direct lookup → timed out → returned → infinite loop. fix: (1) when direct lookup fails (None or exception), mark `audiodb_match_status="error"` so the queue\'s NULL-status filter stops re-picking the row on every tick. preserves the existing `audiodb_id` (no fallback to name-search guess that would overwrite a manual match). (2) extended the retry-after-cutoff queue priorities (4/5/6) to include `\'error\'` rows alongside `\'not_found\'` — same `retry_days=30` window. transient audiodb outages still recover automatically; permanently-broken IDs eventually get re-attempted once a month. only triggered for entities in the inconsistent state of `audiodb_id` set + `match_status` NULL — happy path and already-matched/already-not-found rows unchanged. 5 new tests pin: lookup-returns-none marks error (no infinite loop), lookup-raises-exception marks error, lookup-success preserves happy path, error-row-past-cutoff gets re-picked, error-row-within-cutoff stays skipped.', page: 'tools' }, - { title: 'Docker: Container No Longer Restart-Loops On Bind-Mounted Staging Folder', desc: 'after pulling latest, the container refused to start. logs showed `mkdir: cannot create directory \'/app/Staging\': Permission denied`. cause traced back to the 2026-05-08 image-bloat fix (commit 70e1750) which changed the Dockerfile from `chown -R /app` to a scoped chown on specific subdirs (the recursive chown was duplicating the whole /app tree into a new layer and ballooning image size). side effect: `/app` itself went from soulsync:soulsync to root:root (Docker WORKDIR default), AND `/app/Staging` was left out of both the Dockerfile mkdir + chown list and only created at runtime by the entrypoint script. on rootless Docker / Podman where in-container "root" maps to a host UID, the entrypoint mkdir on `/app/Staging` could fail with EACCES depending on the bind-mount path\'s host ownership — `set -e` then aborted the script and the container restart-looped. fix: (1) Dockerfile now pre-bakes `/app/Staging` into the image alongside the other runtime mount points (mkdir + scoped chown) so the entrypoint mkdir is a guaranteed no-op even when bind-mount perms are weird. (2) entrypoint mkdir + chown both have `|| true` now so any future bind-mount permission quirk surfaces as a log line, not a restart loop. (3) new writability audit at the end of entrypoint setup — `gosu soulsync test -w` on every bind-mountable dir, logs a loud warning with the exact `chown` command to run on the host if perms mismatch the configured PUID/PGID. catches the underlying bind-mount perm issue that the restart-loop fix would otherwise mask (container starts, but auto-import / downloads write into unwritable dirs and fail silently). zero behavior change for users whose containers were already starting fine; defensive against the rootless/podman config that broke after the image-bloat refactor.', page: 'tools' }, - { title: 'Your Albums: Download Missing Now Opens Selectable Modal + Tidal Resolution', desc: 'two-part fix to the your albums "download missing" flow on discover. (1) replaced the broken per-album direct-download loop with a selectable-grid modal mirroring the library page\'s download discography flow. clicking the download button now opens a checkbox grid showing every missing album (cover, title, artist, year, track count, source) with select all / deselect all controls. user picks what they actually want, hits "add to wishlist", each album\'s tracks get resolved + queued through the existing wishlist auto-download processor. matches the discography flow\'s per-album ndjson progress stream so users see ✓/✗ per album as it processes. previous loop fired direct downloads via `openDownloadMissingModalForYouTube` which the user reported as silently failing — "queuing 2/2" toast with no actual transfer activity. wishlist is the right destination for batch missing-album adds since it already handles retry, source fallback, dedup, and rate limiting. (2) added tidal source resolution. backend `/api/discover/album//` got a new `tidal` source branch that calls a NEW `tidal_client.get_album_tracks(album_id)` method — two-phase fetch (cursor-walk `/v2/albums//relationships/items?include=items` for track refs + position metadata, batch-hydrate via existing `_get_tracks_batch` for artist/album names). track refs carry `meta.trackNumber` + `meta.volumeNumber` so multi-disc compilations render in album order. inline `?include=coverArt` lookup pulls the album cover too. single-album click flow (`openYourAlbumDownload`) gets `tidal_album_id` added to `trySources`. virtual-id generation includes tidal_album_id for stable identifiers. backend reuses the existing `/api/artist//download-discography` endpoint — its url artist_id param is functionally unused (per-album payload carries everything), so the modal posts with placeholder `your-albums` and gets multi-artist resolution for free. 10 new tests pin the tidal album-tracks method: single-page walk + hydration, multi-page cursor chain, multi-disc sort order, limit short-circuit, no-token short-circuit, http error returns empty, 429 propagates to rate_limited decorator, forward-compat type filter, partial-batch failure containment, empty-album short-circuit.', page: 'discover' }, - { title: 'AcoustID Scanner: File-Tag Fallback For Legacy Compilation Tracks', desc: 'follow-up to the compilation-album scanner fix. previous patch made the scanner read `tracks.track_artist` (per-track artist column) via COALESCE so compilation tracks would compare against the right value. but tracks downloaded BEFORE that column existed have track_artist=NULL — COALESCE falls back to album artist (the curator) and we\'re back to the wrong-comparison case. fix: explicit 3-tier resolution in `_scan_file` — (1) `tracks.track_artist` from DB if populated → trust it (respects manual edits from the enhanced library view), (2) audio file\'s ARTIST tag via mutagen if present → use it (tidal/spotify/deezer all write the per-track artist into the file at download time, so it\'s ground truth even when DB is stale), (3) album artist → final fallback for files without proper ARTIST tags AND no DB track_artist. file open is essentially free since acoustid is opening it for fingerprinting anyway. critical guard: when DB track_artist is populated (curated value), it always wins over file tag — protects users who edited DB but didn\'t re-tag the file from getting false-positive flags. closes the legacy-data gap without requiring a one-time DB backfill or a re-download. 5 new tests pin: file-tag-resolves-skowl-case (legacy NULL track_artist → file tag wins → no flag), tag-missing-falls-back-to-album-artist (preserves existing genuine-mismatch contract), mutagen-exception-swallowed (debug log, fall-through), tag-matches-DB no behavioral change, and the false-positive guard (DB populated → trumps stale file tag).', page: 'tools' }, - { title: 'Tidal Favorite Albums + Artists Now Show Up On Discover', desc: 'discover → your albums (and your artists) was returning nothing for tidal users regardless of how many albums/artists they\'d favorited. cause: `get_favorite_albums` and `get_favorite_artists` were calling the deprecated `/v2/favorites?filter[type]=ALBUMS|ARTISTS` endpoint, which returns 404 for personal favorites — that endpoint is scoped to collections the third-party app created itself, not the user\'s app-level favorites. the V1 fallback was also dead because modern OAuth tokens carry `collection.read` instead of the legacy `r_usr` scope V1 requires (returns 403). same root cause as the favorited tracks fix from #502. fix: rewire to the working V2 user-collection endpoints — `/v2/userCollectionAlbums/me/relationships/items` and `/v2/userCollectionArtists/me/relationships/items` — using the same cursor-paginated pattern shipped for tracks. ID enumeration lifted into a generic `_iter_collection_resource_ids(path, expected_type, max_ids)` helper so tracks/albums/artists all share one walker (~80 lines deduped). batch hydration via `/v2/{albums|artists}?filter[id]=...&include=...` with extended JSON:API include semantics — single request returns 20 albums + their artists + cover artworks all in `included[]`, parsed via two static helpers (`_first_artist_name`, `_first_artwork_url`) that map relationship refs to the included map. cover/profile images pick `files[0]` (largest variant Tidal returns, typically 1280×1280). public methods preserve the prior return shape so the discover aggregator in web_server.py stays byte-identical. 24 new tests pin: cursor-walker dispatch (correct path + type), included-map building, artist + artwork relationship resolution (full + missing + unknown-id), batch hydration parse for albums + artists, empty-input + HTTP-error short-circuits, BATCH_SIZE chunking (41 IDs → 20/20/1), end-to-end orchestrator behavior.', page: 'discover' }, - { title: 'Server Playlist Sync: Append Mode (Stop Overwriting User-Added Tracks)', desc: 'discord report (cjfc, 2026-04-26): syncing a spotify playlist to your server overwrote anything you\'d manually added to the server-side playlist. now there\'s a per-sync mode picker next to the Sync button on the playlist details modal: "Replace" (default, current behavior — delete + recreate) or "Append only" (preserve existing, only add tracks not already there). useful when the source platform caps playlist size (spotify 100-track limit) and you\'re manually building beyond it on the server. each server client (plex / jellyfin / navidrome) gets a new `append_to_playlist(name, tracks)` method that uses the server\'s native append api — plex `addItems`, jellyfin `POST /Playlists//Items`, navidrome subsonic `updatePlaylist?songIdToAdd=...`. no delete-recreate, no backup playlist created in append mode (preserves playlist creation date + metadata + non-soulsync-managed tracks). dedup-by-id ensures we never add a track that\'s already on the playlist (matched by ratingKey for plex, jellyfin guid id for jellyfin, song id for navidrome — server-native identity, not fuzzy title+artist match). falls back to `create_playlist` when the playlist doesn\'t exist yet (first sync). sync_service dispatches via the new mode flag through /api/sync/start; soulsync standalone has no playlist methods at all so the dispatch falls back to update_playlist with a warning log when append is requested against it. 15 new tests pin: missing playlist → create delegation, dedup filtering (existing ids skipped), short-circuit on no-new-tracks (no api call), failure paths return False without raising, contract listing for each server client.', page: 'sync' }, + '2.5.9': [ + { date: 'May 21, 2026 — 2.5.9 release' }, + { title: 'Now-playing modal: lyrics panel', desc: 'new lyrics panel below the player controls in the expanded now-playing modal. fetches from LRClib via /api/lyrics/fetch, but prefers the local .lrc / .txt sidecar files SoulSync drops next to your audio during post-processing so downloaded tracks show lyrics instantly with zero network. synced LRC (timestamped) highlights the active line and auto-scrolls it into the middle of the viewport on every audio timeupdate; plain text renders without highlighting. status chip shows whether the result came back Synced or Plain. panel is collapsed by default — click the Lyrics header to expand. cached per track so revisiting a track doesn\'t refetch.' }, + { title: 'Now-playing modal: View Artist closes the modal first', desc: 'tapping View Artist on the expanded media player now closes the now-playing modal before navigating, so the artist page is actually visible instead of sitting under a modal you\'d have to manually dismiss. click is a no-op when no artist_id is attached to the current track.' }, + { title: 'Torrent and Usenet release downloads', desc: 'torrent and usenet are now real download sources backed by Prowlarr plus your configured downloader client. album downloads use a release-first staging flow so SoulSync downloads one release, watches live progress, then imports the matching tracks from the staged files. hybrid mode keeps torrent / usenet out of album batches, but still lets them participate for playlist singles and wishlist tracks.' }, + { title: 'Fix: HiFi public instance compatibility', desc: 'HiFi instance probing now understands both supported manifest shapes: the newer /trackManifests-style flow and the public hifi-api /track/ legacy flow. instances that can search and return a playable HLS manifest are no longer mislabeled as Search only. browser-openable pages can still be offline from the API side, so HTTP 502 / Offline labels are still shown honestly.' }, + { title: 'Fix: Jellyfin full refresh imports tracks on older databases', desc: 'full refresh could import artists and albums but fail every Jellyfin track on upgraded databases that were missing newer media columns. startup repair now adds the missing tracks.file_size and albums.api_track_count columns before refresh work runs, so old databases can accept new Jellyfin track rows again.' }, + { title: 'Fix: transient SQLite disk I/O errors retry cleanly', desc: 'cache maintenance and the Tools page full-refresh job now retry short-lived SQLite disk I/O failures around destructive maintenance writes. this targets the reported pattern where the first run failed with disk I/O error and an immediate second run succeeded.' }, + { title: 'Fix: Album Completeness blocks wrong-artist fills', desc: 'artist matching now wins before album/single title matching. if Album Completeness is filling a Jamiroquai track, a same-title release by a different artist is rejected instead of being allowed through by a loose title match.' }, + { title: 'Fix: Album Completeness no longer cross-contaminates artists', desc: 'reported case: filling a Jamiroquai "Light Years" single brought in tracks from Gut\'s "Light Years" album — completely different artist, completely different genre. Root cause: the auto-fill artist gate was a loose 0.50 SequenceMatcher threshold that could let unrelated candidates through. Two new defenses now block this entirely. Stage 1 skips the whole track-fill operation up front if the missing track\'s source artist doesn\'t match the target album artist — compilation albums (various artists / soundtrack) still pass through. Stage 2 replaces the per-candidate 0.50 SequenceMatcher with an alias-aware 0.82 strict matcher that handles diacritics (Beyoncé/Beyonce) and known artist aliases. Both stages logged with structured warnings so future mismatches are diagnosable from the logs.' }, + { title: 'Code review refactor pass on the torrent / usenet flow', desc: 'cleanup commits before review. Lifted the shared album-pick + staging-collision helpers out of torrent.py into a new core/download_plugins/album_bundle.py module so usenet.py no longer reaches into a sibling plugin\'s private surface. Lifted the ~90-line inline album-bundle gate out of run_full_missing_tracks_process into core/downloads/album_bundle_dispatch.py with a clean pure-predicate / inject-deps design so the gate is unit-testable in isolation. Replaced the staging matcher\'s direct download_batches import with an injected get_batch_field accessor on StagingDeps. Made the 6h poll timeout configurable via download_source.album_bundle_timeout_seconds. Added atomic .tmp + rename copy so the Auto-Import worker can never observe a partial audio file during the album-bundle copy loop. 49 new tests across album_bundle, album_bundle_dispatch, and staging-provenance modules pin the contracts.' }, + { title: 'Hybrid mode skips torrent / usenet on album batches', desc: 'follow-up to the album-bundle flow. When download mode is set to Hybrid AND the batch is an album, the per-track search loop now silently strips torrent and usenet from the source chain — they\'re release-level sources that can\'t match per-track meaningfully, and the album-bundle handling only fires in single-source mode. Without the skip, hybrid + torrent-first would fire N redundant Prowlarr searches per album and rely on Auto-Import sweeping Staging to recover. Cleaner now: hybrid falls straight through to per-track-compatible sources (Soulseek / streaming) for albums, and torrent / usenet still get a shot for single-track / wishlist / basic-search use cases where per-track makes sense.' }, + { title: 'Album-bundle flow for torrent / usenet downloads', desc: 'fixes the core architectural problem with indexer-based sources. Prowlarr returns release-level torrents — searching per-track for "Luther (with SZA)" against the GNX album torrent scores near-zero and the orchestrator rejects every candidate. New gated flow: when downloading an album AND torrent or usenet is the single active source (not hybrid), SoulSync now does ONE Prowlarr search for the whole release, picks the best torrent (prefers FLAC, high seeders, reasonable size — drops single-track torrents that snuck in), hands it to your torrent / usenet client, walks the resulting audio files (extracting .zip/.rar/.tar if needed), and drops them all into the staging folder. The existing per-track staging matcher then imports each one to the library by fuzzy title match — same path as the Auto-Import flow. Gate is strictly opt-in: per-track flow is completely untouched for hybrid mode, non-album downloads, and every other source. 5 new tests cover the album picker (seeded-FLAC preference, size floor for single-track torrents, fallback when all candidates are small) and the staging path collision handler.' }, + { title: 'Filesystem-access heads-up for torrent / usenet sources', desc: 'new advisory card on the Indexers & Downloaders tab explaining the cleanest setup: point ALL your downloaders (Soulseek, qBittorrent, SABnzbd / NZBGet) at the same download folder. One folder, one mount, everything just works. Bare-metal needs no change; Docker users can reuse the existing ./downloads mount and just configure each client to write there. docker-compose.yml updated to call this out as the easiest path, with optional commented placeholders for users who prefer separate folders per protocol.' }, + { title: 'Torrent and Usenet downloads', desc: 'two new download sources live in the Download Source dropdown: Torrent Only (via Prowlarr) and Usenet Only (via Prowlarr). they reuse the Prowlarr + torrent client + usenet client you set up on the Indexers & Downloaders tab. searches go through Prowlarr filtered by protocol, picked releases get handed to your torrent client or usenet client, and the resulting files get walked through archive_pipeline (extracts .zip / .rar / .tar when the client didn\'t already do it) and handed to the matching pipeline. both sources are also available in hybrid mode alongside soulseek / youtube / tidal / etc. one caveat: SoulSync needs read access to the torrent / usenet client\'s save_path — works out of the box for everything-on-one-box setups, but remote downloader hosts will need a future sync step.' }, + { title: 'Archive pipeline module (groundwork for torrent / usenet downloads)', desc: 'new core/archive_pipeline.py — walks a directory for audio files (recursive, case-insensitive extensions), extracts zip / tar / tar.gz / rar / 7z archives in-place (rar and 7z are optional deps that warn but don\'t crash if absent), and rejects any archive member trying to escape the destination via path traversal. shared helper the upcoming torrent and usenet download plugins both consume — usenet downloaders usually auto-extract, but the occasional torrent ships an album in a .rar and SoulSync handles it now. 21 unit tests cover the walker + zip + tar extraction + path-traversal protection.' }, + { title: 'Indexers & Downloaders tab restyled with collapsible sections', desc: 'tab now opens with an intro hero card explaining the flow (indexers find releases → downloader fetches them → SoulSync imports) and folds the rest into three collapsible sections: Indexers, Torrent Client, Usenet Client. each section gets a per-service color accent (Prowlarr orange, torrent sky-blue, usenet violet), a status dot in the header (green when Test Connection succeeds, red on failure, grey before testing), and Lidarr-style indexer cards with protocol badges instead of the inline emoji list. Indexers section is open by default; the downloader sections start collapsed since not everyone uses both protocols.' }, + { title: 'Regression tests for the new indexer + downloader plumbing', desc: 'mocked unit tests for Prowlarr + all five downloader adapters (qBittorrent, Transmission, Deluge, SABnzbd, NZBGet). 54 tests pin the state-mapping tables, the parse logic, and the protocol quirks each client needs handled (qBit Referer header, Transmission session-id renegotiation, Deluge magnet-vs-URL method split, SAB queue+history merge, NZBGet 64-bit size fields). next time anyone touches one of these adapters, CI catches breakage before it hits a real downloader.' }, + { title: 'Usenet client adapters (SABnzbd, NZBGet)', desc: 'third commit in the torrent + usenet rollout. SoulSync now talks to SABnzbd and NZBGet through a sibling adapter contract that mirrors the torrent adapter set — pick one downloader in Settings → Indexers & Downloaders, fill in its API key (SABnzbd) or username + password (NZBGet), and Test Connection confirms the link. all three layers are now stood up: Prowlarr finds releases, the torrent adapter and the usenet adapter each know how to ship work to the underlying client. next commit wires Prowlarr search → adapter dispatch → archive extraction so the new sources actually download. job state mapping covers SABnzbd queue + history and NZBGet groups + history, including the verify/repair/unpack phases that are unique to usenet.' }, + { title: 'Torrent client adapters (qBittorrent, Transmission, Deluge)', desc: 'second commit in the torrent + usenet rollout. SoulSync can now talk to any of the three big torrent clients through a single adapter contract — pick which one you use in Settings → Indexers & Downloaders, paste your WebUI URL and credentials, and Test Connection confirms the link. each adapter handles its own auth quirk (qBit cookies, Transmission session-id, Deluge JSON-RPC) and maps native state strings onto a uniform set so the rest of the app stays client-agnostic. no download wiring yet — that gets layered on once the usenet client adapters land in the next commit.' }, + { title: 'Prowlarr integration', desc: 'new Indexers & Downloaders tab in Settings. point SoulSync at your Prowlarr instance with a URL and API key, and you can browse the indexers Prowlarr exposes from inside the app. this is the search half of the upcoming torrent and usenet download sources — wires up the indexer list now so later commits can plug the download flow on top. Lidarr already pulls from its own indexers; Prowlarr unlocks the same search surface to the rest of the download pipeline.' }, ], - '2.5.0': [ - // --- May 10, 2026 — 2.5.0 release --- - { date: 'May 10, 2026 — 2.5.0 release' }, - { title: 'Tidal: Favorite Tracks Now Show Up As A Playlist (Same As Spotify Liked Songs)', desc: 'github issue #502 (yug1900): tidal users wanted their favorited tracks ("my collection" in the tidal app) to appear alongside their normal playlists in the sync tab — same treatment spotify gets for "liked songs". prior attempt at this surfaced empty data because the wrong endpoint was being hit (`/v2/favorites?filter[type]=TRACKS` returns nothing for personal favorites — that endpoint is scoped to collections the third-party app created itself, not the user\'s app-level favorites). reporter located the working endpoint: `GET /v2/userCollectionTracks/me/relationships/items?countryCode=US&locale=en-US&include=items`. cursor-paginated (20 per page, follow `links.next` with `page[cursor]=...` until exhausted), responses only carry track-level attributes — artist + album NAMES come back as relationship-link stubs, not embedded data. fix: two-phase fetch. phase one walks the cursor chain to enumerate every track id (cheap, IDs only). phase two batch-hydrates 20 IDs at a time through the existing `_get_tracks_batch` helper which already knows how to `include=artists,albums` and produce fully-populated `Track` objects matching the rest of the codebase — no duplication of the JSON:API artist/album parse, no new dataclass shape. virtual playlist `tidal-favorites` appended to the end of `/api/tidal/playlists` (mirrors spotify\'s liked-songs placement). id intentionally has NO colon — sync-services.js renderer interpolates ids into css selectors via template literals (`#tidal-card-${p.id} .foo`) and a colon would parse as a css pseudo-class operator. `tidal_client.get_playlist("tidal-favorites")` recognizes the virtual id and dispatches to the collection path internally, so every existing per-id consumer gets it for free: per-playlist detail endpoint, mirror auto-refresh automation, "build spotify discovery from tidal playlist" flow. needs token reconnect to grant the new `collection.read` oauth scope (added to the auth flow). existing tokens hit a 401 — the client now sets a `_collection_needs_reconnect` flag and the listing endpoint surfaces a placeholder card titled "Favorite Tracks (reconnect Tidal to enable)" with a description pointing at settings, so the user has something visible to act on instead of a silently missing row. 22 new tests pin the cursor walk (full chain, max-ids cap mid-page + at page boundary), auth gates (no token / 401 / 403 all bail clean), reconnect-flag lifecycle (set on 401/403, cleared on next successful walk, NOT set on 5xx so transient server errors don\'t falsely tell the user to reconnect), forward-compat type filter (non-track entries skipped), count helper, batch hydration delegation + chunking at the 20-per-batch cap, partial-batch failure containment, and the virtual-id dispatch (real playlist ids still flow through the normal path).', page: 'sync' }, - { title: 'Library Reorganize: Stop Leaving Orphan Audio Files Behind + Hint For Unknown-Artist Rows', desc: 'discord report (foxxify): library reorganize wasn\'t organizing everything. two distinct gaps. (A) lossy-copy users have `track.flac` AND `track.opus` side-by-side at the source; the db only knows about ONE of those (whichever is the canonical library entry). reorganize moved the canonical, left the other format orphaned at the old location, and the empty-folder cleanup never fired because the source dir still had audio in it. fix: at the per-track finalize step the reorganize code now scans the source dir for sibling-stem audio files (same filename stem, audio extension, different format), moves them to the same destination dir as the canonical with the renamed stem + their original extension, then proceeds with the existing source removal + cleanup. preserves both formats post-move so users keep their flac archive AND their opus library copy. (B) old "Unknown Artist / album_id / 0 tracks" rows left over from the pre-#524 manual-import bug couldn\'t be relocated because the album row has no usable metadata source id — reorganize emitted a generic "run enrichment first" message that doesn\'t apply (enrichment can\'t fix these rows; they need their real metadata recovered from file tags). these are the existing `Fix Unknown Artists` repair job\'s domain — reads file tags, re-resolves artist/album/track via configured metadata source, re-tags + moves. reorganize now detects the bad-metadata shape (Unknown Artist OR album.title that\'s a 6+ digit numeric id) and emits a clear "run the Fix Unknown Artists repair job to recover real artist/album from file tags first" hint instead, pointing the user at the right tool. fixer was already implemented and handles the case end-to-end — discoverability gap, not a logic gap. 31 new tests pin: orphan-format detection (canonical-vs-sibling, multi-format, defensive on missing source dir, sidecar exclusion), sibling-move with renamed-stem propagation + dst-dir creation + idempotent re-runs + os-failure handling, and the unknown-artist-hint detection helpers (placeholder names, numeric-id title detection at 6+ digit cutoff, real-album-with-no-source-id keeps the generic enrichment hint, strict-source mode preserved when artist/title look fine).', page: 'library' }, - { title: 'AcoustID Scanner: Compilation Albums No Longer Flag Every Track', desc: 'discord report (skowl): downloaded a compilation album like "high tea music: vol 1" where every track has a different artist (eclypse, andromedik, t & sugah, gourski, himmes, sektor, lexurus, etc.) and the acoustid scanner flagged every single track as wrong song — the file tag had the correct per-track artist (e.g. "eclypse" for "city lights") but the scanner compared against the album-level artist ("andromedik", the curator). raw similarity 12% → wrong song flag. the multi-value-credit fix from the prior pr (foxxify) didn\'t help because both sides were single-value but DIFFERENT artists. cause: scanner sql joined `artists` table via `tracks.artist_id` which points at the ALBUM artist, not the per-track artist. but `tracks.track_artist` column was already populated with the correct per-track value by every server scan + auto-import path that handles compilations. scanner just wasn\'t reading it. fix: changed the scanner select to `COALESCE(NULLIF(t.track_artist, \'\'), ar.name)` — prefers per-track artist when populated, falls back to album artist for legacy rows / single-artist albums where track_artist is null. NULLIF handles the empty-string-instead-of-null case for legacy data. composes with foxxify\'s multi-value fix — for the rare compilation track where acoustid ALSO returns a multi-value credit, both paths work together. 2 new tests pin: compilation track uses per-track artist (reporter\'s exact case), null/empty track_artist falls back to album artist via coalesce.', page: 'library' }, - { title: 'AcoustID Scanner: Multi-Artist Songs No Longer Flagged As Wrong', desc: 'discord report (foxxify): the acoustid scanner repair job was flagging multi-artist tracks as "wrong song" because acoustid returns the full credit ("okayracer, aldrch & poptropicaslutz!") while the library db carries only the primary artist ("okayracer"). raw similarity scored ~43% — well below the 60% threshold — so the scanner created a wrong-song finding even though the audio was correct. user couldn\'t fix without lowering the global artist threshold to ~30% (which would let real mismatches through). cause: scanner used raw `SequenceMatcher` comparison that doesn\'t recognise the primary artist is just one of several contributors in the credit string. fix: extended the shared `core/matching/artist_aliases.py::artist_names_match` helper (lifted in #441) with credit-token splitting on common separators (comma, ampersand, semicolon, slash, plus, "feat.", "ft.", "featuring", "with", "vs.", "x"). when actual artist contains separators, helper splits into individual contributors and checks each against expected — primary-in-credit cases now resolve at 100% instead of 43%. composes with existing alias path so cross-script multi-artist credits ("hiroyuki sawano" expected, "澤野弘之, featured" actual) work too. wired into `core/repair_jobs/acoustid_scanner.py` — replaces the raw similarity call. acoustid post-download verifier already used the helper from #441 so it inherits the same fix automatically. 14 new tests pin: split-by-separator across 12 credit-string formats, primary at start/middle/end of credit, no-mask on genuine mismatches, single-token actual falls through to direct compare, multi-value composes with aliases, threshold still respected, end-to-end scanner integration with reporter\'s exact case (okayracer in okayracer-aldrch-poptropicaslutz credit → no finding), end-to-end scanner still flags genuine mismatches.', page: 'library' }, - { title: 'Deezer Cover Art: Embedded Covers No Longer Look Blurry', desc: 'discord report (tim): downloaded cover art via deezer metadata source came out visibly blurry in navidrome and on phones — particularly noticeable on large displays. cause: deezer\'s api returns `cover_xl` urls at 1000×1000 but the underlying cdn serves up to 1900×1900 by rewriting the size segment in the url path. soulsync wasn\'t doing the rewrite — same as iTunes mzstatic and spotify scdn already get upgraded. now `_upgrade_deezer_cover_url` (mirrors `_upgrade_spotify_image_url` pattern) rewrites the cdn url to request 1900×1900 before download. cdn serves source-native size when source < target so asking for 1900 on smaller-source albums returns the same bytes (no upscaling, no failure). applied at both download sites — auto post-process flow + the enhanced library view\'s "write tags to file" feature. existing `prefer_caa_art` toggle in settings → library → post-processing remains as the orthogonal workaround for users who want even higher quality (musicbrainz cover art archive, often 3000×3000+). 16 new tests pin: standard upgrade, alternate dzcdn host, artist picture urls, custom target sizes, idempotency on already-upgraded urls, defensive on non-deezer urls (spotify/itunes/caa/lastfm/random), empty/none handling.', page: 'settings' }, - { title: 'Cross-Script Artist Names No Longer Quarantine Files (Hiroyuki Sawano / 澤野弘之, Сергей Лазарев / Sergey Lazarev)', desc: 'github issue #442 (afonsog6): files where the artist tag was in one script and the expected metadata was in another — japanese kanji `澤野弘之` for `hiroyuki sawano`, cyrillic `сергей лазарев` for `sergey lazarev`, etc. — got quarantined post-download because acoustid verification scored the artist similarity at 0% (the two scripts share no characters). reporter could not even rescue the file via manual import — the import-modal goes through the same verifier and re-quarantined the same file. cause: verifier compared expected vs actual artist with raw `_similarity` and never consulted musicbrainz aliases, even though MB exposes them on every artist record. fix: new `core/matching/artist_aliases.py` pure helper with alias-aware comparison + new `artists.aliases` JSON column populated by the existing MB enrichment worker on every artist match (one extra `inc=aliases` request per artist) + new multi-tier resolver `MusicBrainzService.lookup_artist_aliases` (library DB → cache → live MB) so the verifier finds aliases even for un-enriched artists without thrashing the MB API. verifier resolves aliases ONCE per `verify_audio_file` call and feeds them through three artist comparison sites (best-match scoring, secondary scan when title matches but artist doesn\'t, final fallback scan). reporter\'s exact two cases reproduced as regression tests with stubbed MB service. backward compat: aliases unavailable / MB unreachable → verifier falls back to direct similarity (identical to pre-fix behaviour — never quarantines stricter than today). 70 new tests pin every layer: pure helper (28), service methods (31), verifier integration (11). audited adjacent artist-comparison sites (auto-import single-track id, discovery scoring, matching engine) — left untouched per scope discipline since they aren\'t the user-reported pain.', page: 'downloads' }, - { title: 'Plex: Library Scan Trigger No Longer Fails On Non-English Section Names', desc: 'github issue #535 (adrigzr): plex servers with the music library named anything other than "music" — Música, Musique, Musik, Musica, etc. — got a `Failed to trigger library scan for "Music": Invalid library section: Music` error after every import cycle, and `wishlist.processing` kept reporting "missing from media server after sync" for tracks that DID import correctly because the post-import scan never fired. cause: `trigger_library_scan` and `is_library_scanning` ignored the auto-detected `self.music_library` (correctly populated by `_find_music_library` filtering by `section.type == "artist"`) and called `self.server.library.section(library_name)` with a hardcoded "music" default — raised NotFound on any non-english server. read methods like `get_artists` already routed through `_get_music_sections` so they didn\'t have the bug; this aligns the scan-trigger path with the same resolution. fix: both single-library branches prefer `self.music_library` first, fall back to literal section lookup only when auto-detection hasn\'t run. activity-feed match in `is_library_scanning` also corrected to use the resolved section\'s actual title instead of the unused `library_name` arg — the prior log line read "triggered scan for music" even on Spanish servers. 13 new tests pin: trigger uses auto-detected section across 6 locale variants (Música / Musique / Musik / Musica / 音乐 / موسيقى), backward-compat fallback when music_library is None, explicit library_name kwarg ignored when auto-detected section exists, log line surfaces correct section title, scan-status check uses auto-detected section\'s `refreshing` attr, activity-feed match filters by resolved title (not library_name).', page: 'settings' }, - { title: 'Search For Match: No More Karaoke / Cover / "Originally Performed By" Junk At The Top', desc: 'github issue #534 (radoslav-orlov): typing "dirty white boy" + "foreigner" into the import-modal "search for match" dialog returned karaoke versions, "originally performed by" compilations, and tribute-band cuts ranked above the actual foreigner studio recording in some regions. user had to scroll past 5+ junk results before finding the canonical track. fix: new `core/metadata/relevance.py` helper reranks results locally with cover/karaoke/tribute/re-recorded penalties (multiplier 0.05× — effectively buries) + exact-artist-match boost (1.5×) + variant-tag (live/acoustic/remix/remaster) penalty (0.4×, skipped when user explicitly typed the variant — searching "track (live)" still ranks live versions correctly). applied at the deezer + itunes + spotify search-tracks endpoints so all three sources behave consistently. validated against live deezer api with the actual #534 query: real foreigner head games cut now lands at #1, live versions follow, karaoke / cover / tribute variants drop to positions 11-15. deezer client also gained optional field-scoped query kwargs (`track="X" artist="Y"`) that build deezer\'s advanced search syntax `track:"X" artist:"Y"` for future opt-in callers (e.g. exact-match flows where api-level filtering is more important than ranking) — kept in client but NOT used at the import-modal endpoint after live testing showed the advanced syntax has its own ranking bias (surfaced "(2008 remaster)" instead of the canonical recording). free-text + local rerank is the more reliable combination here. 75 new tests pin every scoring component, pattern detection (13 cover patterns, 11 variant patterns, 3 fields), score composition (real-cut > karaoke > remaster > re-recorded), the issue #534 screenshot reproduced as a regression test, deezer client query construction + free-text fallback safety net.', page: 'import' }, - { title: 'Auto-Import: Album Duration Is Album Total + Re-Imports Fill Metadata Gaps', desc: 'two more parity gaps closed in the soulsync standalone library write path. (1) album row\'s `duration` column was being written with the FIRST imported track\'s duration instead of the album total — pre-existing bug that survived the prior parity commit. soulsync_client deep scan computes `sum(t.duration for t in self._tracks)` for each album; auto-import now mirrors that by computing the sum across every matched track in the worker and threading it through context to the album INSERT. (2) `record_soulsync_library_entry` was insert-only on artists + albums — once a row existed (matched by id OR name fallback), subsequent imports of the same artist or album skipped completely. meant: artist genres / thumb / source-id reflected ONLY whatever the FIRST imported album supplied, never refreshing as more albums by that artist landed (ten more deezer/spotify imports later, artist row still had whatever the first random import wrote). new conservative UPDATE path: when an existing row matches, fill ONLY the columns whose current value is NULL or empty — never overwrites populated values. protects manual edits + enrichment-worker writes the same way scanner UPDATEs preserve enrichment columns. f-string column names are validated against an allowlist (`_SOULSYNC_FILLABLE_COLUMNS`) before interpolation — defensive against accidental misuse adding columns without an allowlist update. 4 new tests pin: album duration uses sum not single-track, re-import fills empty thumb + genres on existing artist row, re-import does NOT clobber populated values, re-import fills empty source-id columns when later import has them.', page: 'import' }, - { title: 'Auto-Import: Genre Tags Land On The Artists Row + ISRC/MBID Type Hardening', desc: 'small followup to the standalone-library parity commit. (1) auto-import now reads the GENRE tag from each matched audio file (mutagen easy mode, supports flac / mp3 / m4a) and aggregates the deduped set across the album onto the new artists row\'s genres column. matches what soulsync_client._scan_transfer would have written if you\'d done a fresh deep scan after the import — your imported artists no longer feel hollow compared to plex / jellyfin / navidrome scans. dedup is case-insensitive but preserves original casing + insertion order so the json column reads naturally ("Hip-Hop, Rap, Trap" not "hip-hop, rap, trap"). (2) defensive `str()` cast on the worker\'s isrc + mbid extraction. metadata source clients all coerce to string today via `_build_album_track_entry`, but if a future source ever returned int / None for either id the side-effects layer would crash on `.strip()`. cheap insurance. 3 new tests pin: genre aggregation produces deduped insertion-order list, empty when no GENRE tags, isrc/mbid hostile-type input (int, None) coerced to safe string before propagation.', page: 'import' }, - { title: 'Auto-Import: SoulSync Standalone Library Now Gets Full Server-Quality Rows', desc: 'soulsync standalone is meant to be a full replacement for plex / jellyfin / navidrome — the imported tracks should land in the db with the same field richness a media server scan would write. they weren\'t. the auto-import context dict (the payload it handed to the post-process pipeline) had no `source` field anywhere, so `record_soulsync_library_entry` couldn\'t pick the right source-id column on the new tracks/albums/artists rows. result: every auto-imported track landed with NULL on `spotify_track_id` / `deezer_id` / `itunes_track_id` / etc. — watchlist scans (which match by stable source IDs) couldn\'t recognise these tracks as already in library and would re-download them on the next pass. fixed by threading `identification[\'source\']` onto the top-level context, plus per-recording IDs (`isrc`, `musicbrainz_recording_id`) onto track_info so picard-tagged libraries land their per-recording metadata directly. also extracted the artist source ID from the metadata source\'s search response (`_search_metadata_source` and `_search_single_track` now pull `best_result.artists[0][\'id\']`) and threaded it through identification → context → standalone library write, so the artists row finally gets its source-ID column populated instead of staying NULL forever. also added `_download_username=\'auto_import\'` so library history shows "Auto-Import" instead of mislabeling every staging import as "Soulseek" (the fallback default), and an "auto_import" → "Auto-Import" mapping in the source-map dicts at side_effects.py to honour it. record_soulsync_library_entry tracks INSERT now also writes `musicbrainz_recording_id` + `isrc` columns directly (matches the navidrome scanner write path). 17 new tests pin: auto-import context carries source for every metadata source (spotify/deezer/itunes/discogs), `_download_username=auto_import`, isrc + mbid pass-through to track_info, album-id back-reference on track_info, artist source-id flows from identification → context (and not from album_id, the prior copy-paste bug), `_search_metadata_source` extracts artist_id from search response, soulsync library writes mbid + isrc to dedicated columns, deezer source maps to deezer_id column, library history + provenance use Auto-Import / auto_import labels.', page: 'import' }, - { title: 'Auto-Import: Process Multiple Albums At Once', desc: 'auto-import used to process one album at a time. drop 5 albums into staging → wait for the first to fully finish (identify + match + every track post-processed) before the second one even starts. on a slow network or with a big batch this means 30+ minutes of staring at "Processing AlbumOne" while the others sit untouched. now there\'s a small bounded thread pool (3 workers by default, configurable) — up to 3 albums process in parallel, the queue moves through the rest as workers free up. clicking "Scan Now" multiple times no longer spawns extra unbounded scan threads — every trigger (timer + manual button) routes through one shared scan lock so duplicate triggers no-op instead of stacking up. live progress widget on the auto-import card now lists EACH in-flight album with its own track index/total/name instead of one shared scalar that the parallel workers used to stomp on each other. graceful shutdown: stopping the worker waits for in-flight pool work to finish before reporting stopped — no half-moved files or partial DB writes mid-album. stats counters (`scanned` / `auto_processed` / `pending_review` / `failed`) now use a lock so parallel workers don\'t lose increments under load. 17 new tests pin: pool size config, scan lock dedup, executor dispatch + bounded parallelism, cross-trigger candidate dedup, graceful shutdown, per-candidate UI state isolation across parallel workers, stats counter thread-safety, and snapshot consistency.', page: 'import' }, - { title: 'Manual Search In The Failed-Track Candidates Modal', desc: 'when a download fails or returns "not found" the user can already click the status cell to open a modal showing whatever search candidates the auto-search left over and pick a different one. that modal now ALSO has a manual search bar. type any query, hit search, get a fresh round of results from the download sources without having to start the whole download flow over from the search page. solves the case where the auto-query was bad (featured artist not in title, parentheticals like "(remastered 2019)" tripping the matcher, slight artist-name variants) but the file genuinely exists on the source. source picker is smart per download mode: single-source mode (soulseek-only / youtube-only / etc) shows a "searching X" label, no dropdown; hybrid mode shows a dropdown with "all sources" default plus every configured source — picking "all" runs parallel searches across all of them and tags each result row with its source badge. only configured sources show up; unconfigured ones are hidden. results stream in as each source completes via NDJSON instead of blocking on the slowest source — the table starts populating the moment the first source returns. clicking a result reuses the existing retry-download flow → same path, same acoustid verification on the file when it lands, no shortcut around the safety net. additive in the truest sense: the existing modal layout / candidates table / download buttons are byte-identical when the user doesn\'t use manual search. backend extends the candidates endpoint with `download_mode` + `available_sources` + a `source` field per candidate (purely additive — old fields untouched), and adds a new `POST /api/downloads/task//manual-search` that streams NDJSON (one header line, one source_results line per source as completed, one done terminator) so the frontend renderer can append rows incrementally. 11 tests pin the streaming contract: query length / source whitelist / task 404 validation, single-source dispatch, parallel "all" dispatch, one-event-per-source streaming shape, unconfigured-source skip + reject, header metadata, and per-source exception isolation (one source raising emits a `source_error` event but doesn\'t fail the stream).', page: 'downloads' }, - { title: 'Manual Picks Don\'t Auto-Retry Anymore (And The Modal Always Opens)', desc: 'three follow-on fixes to the manual-search feature once people started actually using it. (1) when the user picked a candidate and that download failed (e.g. soundcloud 404 on a stale track url), the auto-retry monitor would treat it like any other failed auto-attempt — yank the task back to "searching" and pick a different candidate. felt completely wrong from the user\'s perspective: "i picked THIS one, why is it searching for something else?" now manual picks are tagged with a `_user_manual_pick` flag and the auto-retry path bails on it. failure surfaces to the user instead of getting silently fallen-back. (2) non-soulseek manual picks (youtube / tidal / qobuz / hifi / deezer / soundcloud / lidarr) were getting stuck at "downloading 0%" forever even after their engine reported terminal failure. cause: status polling went into a "let monitor handle retry" branch that never fired because manual picks bail on retry — task was orphaned in downloading state. fix: when the engine reports Errored on a manual pick, mark the task failed directly, don\'t defer to the monitor. plus an engine-state fallback path covers the rare race where the orchestrator\'s pre-populated transfer lookup is missing the entry. (3) failed / not_found rows were only clickable when the auto-search had cached candidates — but the whole point of opening the modal now is to RUN a manual search, which doesn\'t need pre-existing candidates. now every failed / not_found / cancelled row opens the modal regardless. (4) one nasty deadlock fix in the process: the new "mark failed" path was synchronously calling `on_download_completed` while holding `tasks_lock`, which itself re-acquires the same lock — `threading.Lock` is non-reentrant so the polling thread wedged forever. while wedged the lock was held → every other endpoint that needed it (including /candidates → can\'t open OTHER modals) hung waiting. moved completion callbacks onto a daemon thread so the lock releases first. (5) manual download worker now runs on its own dedicated thread instead of competing with the batch\'s 3-worker `missing_download_executor` pool — saturated batches no longer queue manual picks indefinitely. all changes are scoped to manual picks only via the `_user_manual_pick` flag — auto-attempt flow is byte-identical to before. 17 unit tests pin the gate behavior (status engine fallback / monitor retry skip / IF-branch failure transition / auto-attempt skip).', page: 'downloads' }, - { title: 'Manual Import: Stop Writing "Unknown Artist / album_id / 0 tracks" Garbage', desc: 'github issue #524 (radoslav-orlov): clicking an album in the import page → all imported albums landed in the library as "Unknown Artist" with the raw 10-digit album id as the title and 0 tracks. cause: the click handler `importPageSelectAlbum(albumId)` was passing only the id to the `/api/import/album/match` POST. the search response carried `source` (which metadata source the album_id came from) + `album_name` + `album_artist`, but the click discarded everything except the id. backend `get_artist_album_tracks` then guessed the source via the configured primary-source priority chain — for a non-deezer-primary user clicking a deezer search result, the chain tries spotify/itunes/discogs first against a deezer numeric id, all return None, and the lookup falls through to the failure-fallback dict (`name = album_id`, no artist field, `total_tracks = 0`). that broken metadata then flowed through the import pipeline → soulsync standalone library got the garbage rows. fix: cache album lookup by id when the suggestions / search renderers run, then have `importPageSelectAlbum` pull `source` + `name` + `artist` from the cache and include them in the match POST. backend now also logs a clear warning when source is missing from the match request, so any future caller dropping it shows up in app.log instead of silently corrupting library imports.', page: 'import' }, - { title: 'Auto-Import: Multi-Disc Albums No Longer Lose Half The Tracks', desc: 'caught while testing #524 with kendrick lamar mr morale & the big steppers (3 discs). dropped discs 1+2 loose in staging root + disc 3 in its own folder, all perfectly tagged → only 9 tracks ended up imported, the rest got integrity-rejected and quarantined. two related bugs in `auto_import_worker._match_tracks`: (1) the "quality dedup" loop kept `seen_track_nums[track_number] = file` and dropped any later file with the same number as a quality duplicate. on a multi-disc release where every disc has tracks 1..N, that collapses the album to one disc\'s worth of files BEFORE the matcher even runs. fix: dedup keys on `(disc_number, track_number)` tuples instead. (2) the 30% track-number bonus in the match scoring fired whenever `ft[track_number] == track_num` regardless of disc — file tagged (disc=2, track=6, "Auntie Diaries") got the full bonus matching API track (disc=1, track=6, "Rich Interlude"), wrong file → integrity check correctly rejected and quarantined. fix: 30% bonus only when BOTH disc and track numbers agree, with a small consolation bonus for cross-disc collisions so title similarity has to carry the match. 4 new tests pin: dedup preserves all files across discs (18-file regression case), match scoring pairs to correct disc, single-disc albums still match normally, and quality dedup within a single (disc, track) position still picks the higher quality file.', page: 'import' }, - { title: 'Auto-Import: Picard / Beets Tagged Libraries Now Get Perfect Matches', desc: 'follow-on to the multi-disc fix. brought the auto-import matcher up to picard / beets / roon parity — files with per-recording identifiers (musicbrainz id or isrc) now match via exact-id lookup before any fuzzy scoring runs. picard-tagged libraries land every track on the first pass with full confidence, no fuzzy guessing. three layered phases now: (1) MBID exact match — file has `musicbrainz_trackid` tag, source returns the same id → instant pair, full confidence. picard\'s primary identifier. (2) ISRC exact match — file has `isrc` tag, source returns the same id → same fast-path, slightly lower priority than mbid (isrc can be shared across remasters of the same recording). (3) duration sanity gate — files in the fuzzy phase whose audio length differs from the candidate track\'s duration by more than 3s are rejected before scoring runs, regardless of how good the title agreement looks. defends against cross-disc / cross-release / wrong-edit mismatches the post-download integrity check used to catch only AFTER the file had already been moved + tagged + db-inserted. metadata-source layer (`_build_album_track_entry`) also extended to propagate isrc + mbid from raw track responses (spotify uses `external_ids.isrc`, itunes uses top-level `isrc`) — without this, fast paths would never trigger in production even though unit tests pass. 18 new tests pin: mbid + isrc exact matches with normalization (dashes / spacing / case), mbid > isrc priority, fast-path bypassing fuzzy scoring entirely, duration gate rejecting wrong-disc collisions, deezer-seconds-vs-spotify-ms duration unit conversion, full picard-tagged 10-track album matching via mbid only.', page: 'import' }, - // --- May 8, 2026 — patch release --- - { date: 'May 8, 2026 — 2.4.3 release' }, - { title: 'Discover: Sharper Track Selection (Diversity, Source-Aware Popularity, Library Dedup, SQL Genre Filter)', desc: 'four selection-quality fixes on the soulsync-made discover playlists. (1) hidden gems and discovery shuffle had no diversity caps — they could return 50 tracks from the same artist or 20 from one album. now both apply the existing `_apply_diversity_filter` (over-fetch 3x then enforce per-album/per-artist caps; shuffle uses tighter caps because it should feel maximally varied). (2) `popularity` thresholds were spotify-shaped (0-100 scale, popular >= 60 / hidden < 40), but deezer writes its rank value into that column (often six-digit integers) and itunes writes nothing meaningful. for deezer-primary users this meant popular picks pulled essentially everything and hidden gems pulled nothing. new `_get_popularity_thresholds(source)` returns per-source values: spotify (60, 40), deezer (500_000, 100_000) ballpark, itunes/other (None, None) which skips the popularity filter entirely and falls back to random + diversity. (3) `get_genre_playlist` used to load up to 1M discovery_pool rows into python and run a substring keyword filter on the json column. now the keyword OR chain pushes down into sql via `(artist_genres LIKE ? OR ...)` placeholders, fetch_limit drops to limit*10. parent-genre expansion via GENRE_MAPPING preserved. (4) discovery selectors now exclude tracks the user already owns — `_select_discovery_tracks` gained `exclude_owned: bool = True` (default on) which adds a `NOT EXISTS (SELECT FROM tracks WHERE source_id matches)` correlated subquery covering the spotify/itunes/deezer-id columns (with the deezer-column-name asymmetry handled inline: discovery_pool.deezer_track_id vs tracks.deezer_id). hidden gems / shuffle / popular picks / decade / genre browser all benefit automatically. 12 new tests (27 total in the file): diversity caps, source-aware threshold values, threshold-skip behavior, sql-pushed genre filter, parent-genre expansion, owned-track exclusion, opt-out flag, and the deezer-column-name asymmetry trap. 2232/2232 full suite green.', page: 'discover' }, - { title: 'Discover: Stop Showing Undownloadable Tracks (+Lift +Cleanup)', desc: 'audit found multiple discover-page sections (hidden gems / discovery shuffle / popular picks / decade browser / genre browser) had no `WHERE (spotify_track_id IS NOT NULL OR itunes_track_id IS NOT NULL OR ...)` gate on their selection sql. tracks with no source ids in the discovery pool were getting displayed, the user would click download, and the download would silently fail because there was nothing to look up. fix: lifted all five discovery_pool selection methods into shared private helpers (`_select_discovery_tracks`, `_apply_diversity_filter`, `_compute_adaptive_diversity_limits`) on `PersonalizedPlaylistsService`. mandatory id-validity gate is hard-coded into the selector — no opt-out flag, every public method inherits it for free. behavior preserved: same diversity tiers, same over-fetch multipliers, same popularity thresholds, same blacklist filter. ~314 lines of repeated select/diversity boilerplate collapsed across the 5 methods (-55% on those methods\' business logic). also deleted four sections that had been stubbed returning [] for ages (recently added / top tracks / forgotten favorites / familiar favorites) — frontend, backend endpoints, html blocks, helper docs, all gone. on the frontend, lifted the duplicated decade-browser + genre-browser tab management (~314 lines of identical fetch-tabs / render-tabstrip / fetch-content / render-tracklist / wire-sync-button code) into one shared `createTabbedBrowserSection(config)` helper. each browser is now a thin wrapper: ~3 lines per public function. 14 new tests pin the gate (every selector filters null-id rows), the diversity caps, the adaptive limit tiers, the source filter, and the blacklist filter.', page: 'discover' }, - { title: 'Internal: Discover Controller — Cin Pre-Review Polish', desc: 'tightened the controller before opening the PR. (1) dropped the magic `extractItems` defaults — controller used to auto-pull `data.items` / `data.albums` / `data.artists` / `data.tracks` / `data.results` if no extractor was provided. removed the fallback chain. each section now MUST supply its own `extractItems(data) => array` callback. cin standard: explicit > implicit; the auto-fallback could silently grab the wrong key on endpoints that return multiple arrays. validated at register-time so misuse fails immediately. all 10 existing call sites already had explicit extractors so no migration churn. (2) replaced the `renderItems` returning null convention (used by Your Albums + manualDom-style sections) with an explicit `manualDom: true` config flag. clearer intent at the call site, less likely to be confused with a renderer error. (3) added a minimal node `--test` JS test file at `tests/static/test_discover_section_controller.mjs` — 32 tests pin the lifecycle contract: config validation (every required field), happy-path fetch+render, empty/stale/error states, no-fetch `data:` mode, manualDom mode, callable `fetchUrl`, load coalescing, refresh bypass, hook error containment, error toasts. runs via `node --test tests/static/` directly, OR via the regular pytest sweep (`tests/test_discover_section_controller_js.py` shells out to node and asserts a clean exit). skipped gracefully when node isn\'t available or is < 22. closes the "controller is a contract, pin it at the test boundary" gap that cin would have flagged. 2205/2205 full suite green (was 2204 + 1 new pytest wrapper); 32/32 node --test pass; ruff clean; js parses clean.', page: 'discover' }, - { title: 'Internal: Discover Cleanup Round — Toast Errors, Stale State, Skipped Sections', desc: 'follow-up to the controller migration. extended `createDiscoverSectionController` with the hooks the per-section migrations surfaced as needed: callable `fetchUrl` (resolves the seasonal-playlist recreate-on-key-change hack), no-fetch `data:` mode (lets render-only sections like seasonal albums use the controller without inventing a fake endpoint), `beforeLoad` hook (lets dynamically-inserted sections like because-you-listen-to ensure their container exists before the spinner shows), `onSuccess(data)` hook (cleaner home for sibling header / subtitle / button updates than folding them into renderItems), and an `isStale` / `onStale` / `renderStale` triple for the third render state (data is empty BUT upstream is still discovering — show updating UI + start a poller, instead of the bare empty-state copy). turned on `showErrorToast: true` for every migrated section — section load failures now surface a global toast instead of silently spinning forever or swallowing into console.debug. that\'s the JohnBaumb #369 pattern applied at the UI layer. migrated the two sections that didn\'t fit the original controller contract: `loadYourAlbums` (uses isStale/onStale for stale-fetch UI + onSuccess for subtitle/filters/download-button side-effects + renderItems returning null since it delegates to the existing grid renderer) and `loadSeasonalAlbums` (uses no-fetch data mode since the parent `loadSeasonalContent` already fetched the season payload). also lifted the duplicated decade-tab + genre-tab sync-status block (✓/⏳/✗/percentage) into a `_renderSyncStatusBlock(idPrefix)` helper — two call sites now share one implementation. listenbrainz playlists keep their own block because the semantics differ (matching progress vs download progress). audit found the 13 supposedly-dead hidden sections aren\'t dead at all — they\'re gated on user data (discovery pool, library content, metadata cache) and self-surface when their data exists. removed one orphaned `loadPersonalizedDailyMixes()` call from `blockDiscoveryArtist` — daily mixes is intentionally paused, refreshing it from there was a no-op.', page: 'discover' }, - { title: 'Internal: Migrate 7 More Discover Sections to the Controller', desc: 'follow-up to the foundation commit. migrated fresh tape, the archives, time machine intro carousel, browse by genre intro carousel, seasonal mix, your artists, and because-you-listen-to onto `createDiscoverSectionController`. each one drops its own hand-rolled try/catch + spinner injection + empty-state HTML + error swallow in favor of a config object — controller owns the lifecycle. net 76 lines smaller in discover.js even after adding the per-section render helpers. skipped two sections that don\'t fit the controller\'s single-fetch / single-render-target shape: `loadYourAlbums` (paginated grid + filters, four separate UI elements updated) and `loadSeasonalAlbums` (no fetch — receives pre-fetched data from parent). hidden / dead sections (~13 of them) untouched in this pass — separate audit commit will surface or kill them. controller extension candidates surfaced for follow-up: callable `fetchUrl` (so seasonal playlist doesn\'t need controller-recreate-on-key-change), explicit `isStale` / `onStale` hook (so your-artists doesn\'t fold stale handling into renderItems), `beforeLoad` hook (so because-you-listen-to can let the controller own the dynamic container creation), and a no-fetch `data:` mode (so render-only sections like seasonal albums can use the controller). zero behavior changes — every public load function keeps its name + signature so existing callers, refresh buttons, and dashboard wiring don\'t notice the swap.', page: 'discover' }, - { title: 'Internal: Discover Section Controller Foundation', desc: 'every section on the discover page (recent releases, your artists, your albums, seasonal, fresh tape, the archives, etc) re-implements the same lifecycle by hand: show spinner → fetch endpoint → parse → either render or show empty state or show error → maybe wire post-render handlers → maybe expose refresh. ~30 sections, all subtly drifting — different empty messages, different error handling (some console.debug, some silently swallowed, some leave the spinner spinning forever), different sync-status icons, no consistent error toast. lifted that lifecycle into a shared `createDiscoverSectionController` (renderers stay per-section because section data shapes legitimately differ — album cards vs artist circles vs playlist tiles vs track rows; the controller is the wrapper, not a forced visual abstraction). this commit is the foundation: built the controller + migrated `recent releases` as proof. each remaining section will migrate in its own follow-up commit (keeps reviews small + lets us sequence the work). once everything is on the controller, the discover-page cleanup work (kill 13 dead sections, standardize sync-status icons, add error toasts) becomes single-line registry edits instead of section-by-section rewrites.', page: 'discover' }, + '2.5.8': [ + { date: 'May 20, 2026 — 2.5.8 release' }, + { title: 'Fix: blank artist pages on Python / git-pull installs', desc: 'PR #644 moved the artist detail page behind a TanStack React route. installs that pull from git but never run `npm install && npm run build` ship without the Vite bundle, so the legacy shell saw `/artist-detail//` URLs and bailed — every click left a blank pane. the legacy startup path now parses the URL itself and hands off to the existing artist detail loader, so Python users get artist pages back without needing to build the webui. Docker / built installs still take the React route as before.' }, + { title: 'Fix: downloads marked complete before post-processing finished', desc: 'monitored Soulseek transfers were getting flipped to "successful" the moment slskd reported the file done — before SoulSync had actually run the post-processing worker (find on disk, fingerprint-verify, import to library). a failed import after that point left a phantom "completed" row that never made it into the library. completion now waits for the real post-processing result; if the worker can\'t even be scheduled, the task is marked failed and the batch slot is released so the queue keeps moving.' }, + { title: 'Disk-backed artwork cache', desc: 'image fetches now route through a disk + SQLite cache with hashed URLs, size / mime validation, stale fallback when the upstream is down, and per-image fetch locking so 12 simultaneous requests for the same album cover share one network round-trip. cuts repeat-load latency and survives metadata source rate limits. served from new `/api/image-cache`; `/api/image-proxy` stays as a compatibility shim.' }, + { title: 'Strict-source downloads check duration before pulling', desc: 'Tidal / Qobuz / HiFi / Deezer-DL / Amazon candidates now get a duration-tolerance check before the download starts, using the same tolerance logic post-processing would apply after. tracks whose duration is far enough off the metadata reference to fail the integrity check are skipped at pick time instead of after wasting the download. Soulseek and YouTube unchanged (they don\'t expose reliable pre-download duration).' }, ], - '2.4.2': [ - // --- May 7, 2026 — patch release --- - { date: 'May 7, 2026 — 2.4.2 release' }, - { title: 'Artist Top Tracks: Per-Row + Bulk Download', desc: 'github issue #513 (s66jones): wanted a way to grab an artist\'s top X popular songs without pulling the full discography (the zotify workflow). artist detail page already had a "popular on last.fm" sidebar, but it was display-only — play button per row, no download. now when your primary metadata source is spotify or deezer, that sidebar pulls top tracks via the source\'s native popularity endpoint (spotify `artist_top_tracks` returns 10 per market, deezer `/artist/{id}/top` supports up to 100), each row gets a wishlist-add button on hover, and a "download all" footer button opens the existing wishlist modal with all top tracks pre-loaded. files land in their REAL album folders on disk (not a fake "top tracks" folder) because each track carries its actual album metadata. itunes / discogs / musicbrainz primary still falls back to the existing last.fm playcount display (no popularity ranking on those sources). 10 new tests pin the spotify + deezer client method behavior (auth gate, limit clamping, malformed response handling, spotify-compatible shape conversion).', page: 'library' }, - { title: 'Fix: AcoustID Verification Let Instrumentals Pass As Vocal Tracks', desc: 'discord report (corruption [BWC]): downloads coming through as instrumental versions when the user expected the vocal cut. slipped past acoustid verification because the title-similarity normalizer strips parentheticals and version-suffix tags ("(Instrumental)", "- Live", etc) so legit name variations don\'t false-fail the comparison. side effect: "in my feelings" and "in my feelings (instrumental)" both normalize to "in my feelings", title sim is 1.0, file passes verification despite being the wrong cut. fix: detect the version label on each side BEFORE normalization runs — if expected and matched disagree (one is original, the other is instrumental / live / acoustic / remix / etc), reject as version mismatch. reuses `MusicMatchingEngine.detect_version_type` so post-download verification uses the same patterns the pre-download soulseek matcher already applies (no duplicated regex tables). also gates the secondary fallback scan, so a wrong-version variant in the same fingerprint cluster can\'t win the loop after the best match is rejected. 6 new tests pin the four direction cases (instrumental returned for vocal request → fail, vocal returned for instrumental request → fail, live vs acoustic → fail, matching versions on both sides → pass) plus the original-to-original happy path and the secondary-scan gate.', page: 'downloads' }, - { title: 'Fix: Search Picker Defaulted to Spotify on Non-Admin Profiles', desc: 'github issue #515 (jaruca): admin sets primary metadata source to deezer / itunes / discogs, but every non-admin profile saw spotify as the active source on the search page and global search popover, requiring manual click each time. cause: `shared-helpers.js` resolved the active source by fetching `/api/settings` — that endpoint is `@admin_only` because it returns full config including credentials, so non-admin profiles got 403 and silently fell back to the hardcoded `spotify` default. fix: read from `/status` instead, which is public and already returns `metadata_source` for the dashboard. one-line scope change, behavior preserved for admins (same value, different endpoint), non-admins now see the real configured source.', page: 'search' }, - { title: 'Internal: Stop Swallowing Exceptions Silently', desc: 'github issue #369 (johnbaumb): the codebase had ~300 `except Exception: pass` blocks — and another ~30 bare `except: pass` ones — across web_server.py, every metadata client, every download/import worker, the repair jobs, and most service modules. when one of those paths failed at runtime, the failure was completely invisible: no log line, no telemetry, nothing. you\'d see "downloads stopped working after a few hours" or "enrichment never finishes" and there was nothing to grep for in app.log because the exception had been thrown straight into the void. swept all of them. converted to `except Exception as e: logger.debug(": %s", e)` so failures land in the log with enough context to grep. bare `except:` cases (which also swallow KeyboardInterrupt and SystemExit — actively wrong) got upgraded to `except Exception:` first so ctrl-c works correctly. ~14 cleanup-path sites (atexit handlers, finally-block conn.close calls) were intentionally left silent with explicit `# noqa: S110` comments — logging during shutdown can itself crash because file handles get torn down before the handler fires. and added ruff S110 to the lint config so this pattern fails CI going forward — drift fails at PR review instead of at runtime against a wedged worker thread. zero behavior change to any happy path; just made the failure paths inspectable. test suite (2188 tests) green throughout the sweep.' }, - { title: 'Fix: Repair Job Card "X Findings" Badge Was Misleading After Bulk-Fix', desc: 'discord report: duplicate detector card said "372 findings" and cover art filler said "60 findings", but clicking the findings tab pending filter showed 0 — read like a bug ("findings aren\'t being created"). actual cause: job-card badge displayed `last_run.findings_created` (historical "found in last scan") which doesn\'t reflect current state when those findings have since been bulk-fixed and moved to status="resolved". fix: api response now includes `pending_findings_count` per job (current pending count from a single sql aggregation). badge now shows "X pending" when pending count > 0 (urgent red color), or "X found in last scan" with a muted grey color when pending = 0 but the last scan did surface something. user can tell at a glance whether something needs review vs whether it\'s a historical reminder. 3 new tests pin the per-job pending count helper.', page: 'stats' }, - { title: 'Fix: Downloads Stop After 2-3 Hours (slskd HTTP Timeout)', desc: 'github issue #499 (bafoed): big initial sync of spotify playlists worked for 2-3 hours then downloads silently stopped. 3 active tasks stuck in "searching" state, replaced every ~10 min by different ones, but slskd ui showed no actual searches happening. only fix was restarting the soulsync container — which would buy another 2-3 hours. root cause: `core/soulseek_client.py` constructed `aiohttp.ClientSession()` with no timeout at four sites. when slskd hung on a request (overloaded, transient network blip, internal stall), the http call blocked indefinitely — and the worker thread blocked with it. download executor only has `max_workers=3` for download workers. once 3 worker threads were wedged on hung calls, no further downloads could start. batch-level "stuck detection" (10-min) was correctly marking tasks `not_found` and trying to start replacements, but the executor pool was exhausted — replacements queued forever. fix: bounded `aiohttp.ClientTimeout` (total 120s, connect 15s, sock_read 60s) on every slskd `ClientSession` construction. legitimate metadata calls (search submission, status polls, download enqueue) finish in seconds — slskd doesn\'t stream files through these requests, so the timeout can\'t kill a real operation. when timeout fires, the request raises `asyncio.TimeoutError` which is now explicitly caught + logged + returns None to the caller (treats as a normal failure, same code path as a 5xx response). worker thread unblocks → executor pool stays healthy → downloads keep flowing. 3 new tests pin the timeout config + the `asyncio.TimeoutError` handler so future drift fails at the test boundary instead of at runtime against a wedged executor.', page: 'downloads' }, - { title: 'Fix: Library Reorganize Job Misclassified Album Tracks As Singles', desc: 'github issue #500 (bafoed): library reorganize repair job moved tracks like `Surf Curse/Surf Curse - Nothing Yet (2017)/01 - Christine F.flac` to single-template paths like `Surf Curse/Surf Curse - Christine F/Surf Curse - Christine F.flac`. root cause: the job used `is_album = (group_size > 1)` where `group_size` was the count of tracks for the same album currently sitting in the transfer folder being scanned — when only one track of an album was in transfer (rest already moved to library, or album tags varied across tracks like "Buds" vs "Buds (Bonus)"), each track became a 1-element group → all routed through single template. fix: rewrote the job to delegate to the per-album planner (`core.library_reorganize.preview_album_reorganize` / `reorganize_queue`) — the same planner the artist-detail "reorganize" modal uses. db-driven: the planner knows the album has multiple tracks regardless of how many sit in the transfer folder, so the album-vs-single classification is structurally correct. apply mode delegates to the existing reorganize queue → file move + post-processing + db update + sidecar handling all flow through one code path. only iterates albums for the ACTIVE media server (matches the artist-detail modal\'s scope) — multi-server users (plex + jellyfin etc) won\'t accidentally have the job touch the inactive server\'s files. albums missing a metadata source id get a single "needs enrichment first" finding instead of n per-track "no source" findings. dropped ~500 loc of tag-reading + transfer-walk + template logic that was duplicated against the per-album path. files in transfer with no db entry are now exclusively the orphan_file_detector\'s domain (clean separation). 12 tests pin the delegation contract.', page: 'library' }, - { title: 'Fix: Enrich Honors Manual Album Matches', desc: 'github issue #501 (tacobell444): if you manually matched an album to a specific source ID via the match-chip UI, then clicked "enrich" on that album, the worker would search by name and overwrite your manual match with whatever the search returned (or revert status to "not_found" if it found nothing). reorganize then read the now-wrong id and moved files to the wrong destination. fix: extracted a shared `core/enrichment/manual_match_honoring.py` helper. every per-source enrichment worker (spotify / itunes / deezer / tidal / qobuz) now reads its stored id column at the top of `_process_*_individual` — if present, it fetches via `client.get_album(stored_id)` directly and refreshes metadata without touching the id. fuzzy name search only runs as fallback for never-matched entities. discogs / audiodb / musicbrainz already had inline stored-id fast paths and are left alone. lastfm / genius are name-based and don\'t store ids. cin-shape lift: same fix in 5 workers gets exactly one helper, per-worker variability (column name, client method, response shape) plugs in via callbacks. 11 new helper tests pin: stored-id fast-path, no-id fallthrough, fetch-failure fallthrough, table/column whitelist, callback contract.', page: 'library' }, - { title: 'Fix: "no such table: hifi_instances" When Adding HiFi Instance', desc: 'github issue #503 (hadshaw21): adding a hifi instance via downloader settings popped up `no such table: hifi_instances` even though the connection test and "check all instances" both worked. root cause: `_initialize_database` runs every CREATE TABLE + every migration step inside one sqlite transaction. python\'s sqlite3 module doesn\'t autocommit DDL by default, so if any later migration step throws on a user\'s specific DB shape (e.g. an old volume from a prior soulsync version with quirky schema state), the WHOLE batch rolls back — including the hifi_instances CREATE that ran successfully. user\'s next boot retries init, hits the same migration failure, rolls back again. table never lands. fix: defensive lazy-create. every hifi_instances CRUD method now runs `CREATE TABLE IF NOT EXISTS hifi_instances (...)` immediately before its operation. idempotent — costs one PRAGMA-level no-op when the table is already present, fully recovers from a broken init. read methods (`get_hifi_instances`, `get_all_hifi_instances`) now return empty instead of raising when init failed. write methods (`add`, `remove`, `toggle`, `reorder`, `seed`) work end-to-end. doesn\'t paper over the underlying init issue (still worth tracking down which migration breaks for which users) but makes hifi instance management self-healing. 7 new tests pin the lazy-create behavior — every method works against a DB that\'s missing the table.', page: 'settings' }, - { title: 'Plex: "All Libraries (Combined)" Mode', desc: 'github issue #505 (popebruhlxix): users with multiple plex music libraries (e.g. one per plex home user) only saw one library inside soulsync because the connection settings forced you to pick a single library section. now there\'s a new "all libraries (combined)" option in settings → connections → plex → music library dropdown. picking it flips the plex client into a server-wide read mode where every read method (`get_all_artists` / `get_all_album_ids` / `search_tracks` / `get_library_stats` / etc) dispatches through `server.library.search(libtype=...)` instead of querying a single section. one api call, plex handles the aggregation. cross-section dedup applied at the listing layer — same-name artists across sections collapse to a canonical entry (the one with more tracks), so plex home families with overlapping music tastes don\'t see "drake" twice. removal-detection id enumeration stays raw on purpose — deduping there would falsely prune tracks linked to non-canonical ratingKeys. write methods (genre / poster / metadata updates) are unaffected and operate on plex objects via ratingKey directly — write-back targets one section\'s copy of an artist if it exists in multiple, document and revisit if it matters. trigger_library_scan + is_library_scanning fan out across every music section in the new mode. backward compatible — existing users with a real library name saved see no behavior change. the "all libraries" option only appears in the dropdown when more than one music library exists on the server. 29 new tests pin both modes (single-section preserved, all-libraries dispatches through server-wide search, dedup keeps canonical, id enumeration stays raw).', page: 'settings' }, - { title: 'Fix: Download Discography Showed Wrong Artist\'s Albums', desc: 'clicked download discography on 50 cent → modal showed young hot rod\'s albums. clicked weird al → modal showed the beatles. cause: the endpoint received whichever single artist id the frontend happened to pick (spotify or itunes or deezer or library db id) and dispatched it as-is to whichever source it queried. when the picked id didn\'t match the queried source\'s id format, lookup either returned wrong-artist results (numeric collisions — db id 194687 was a real deezer artist for someone else) or fell back to a fuzzy name search that picked a wrong artist. the per-source id dispatch mechanism (`MetadataLookupOptions.artist_source_ids`) already existed and the watchlist scanner already used it; the on-demand discography endpoint just wasn\'t wired to it. fixed: when the url artist_id matches a library row by ANY stored id (db id, spotify_artist_id, itunes_artist_id, deezer_id, musicbrainz_id), backend pulls every stored provider id and dispatches the right id to each source. each source gets its OWN stored id regardless of what the url carries. when the url id is a non-library source-native id and the row lookup misses entirely, behavior is identical to before (single-id fallback). also fixed two log-namespace bugs: enhance quality and multi-source search were writing through `getLogger(__name__)` which resolves to `core.artists.quality` / `core.metadata.multi_source_search` — neither under the soulsync handler — so every diagnostic line was silently dropped. switched both to `get_logger()` from utils.logging_config so they actually land in app.log.', page: 'library' }, - { title: 'Enhance Quality: Direct ID Lookup Like Download Discography', desc: 'two related fixes. (1) discord report: enhance quality on an artist with neither spotify nor deezer connected added tracks as "unknown artist - unknown album - unknown track". enhance was running a single-source itunes fallback chain that returned junk matches with empty fields, while track redownload had been doing parallel multi-source search the whole time. extracted that search into `core/metadata/multi_source_search.py` and pointed both flows at it. (2) followup: enhance was still using fuzzy text search even when the library track had a stored source ID (spotify_track_id / deezer_id / itunes_track_id / soul_id) on the row, which meant tracks with messy tags ("Title (Live)", featured artists in the artist field, etc.) failed to match even though a perfect ID was sitting right there. download discography never had this problem because it resolves albums by stable ID, not by name. enhance now does the same: for every source you have configured, if the track has a stored ID for that source, it calls `get_track_details(id)` directly — no fuzzy matching. preferred source (your configured primary) is tried first so a deezer-primary user gets deezer payloads on the wishlist entry. text search is only the fallback now (kicks in for tracks with no stored IDs). also fixed the modal toast that lied "matching tracks to spotify..." regardless of which sources were actually being queried.', page: 'library' }, - { title: 'Internal: Media Server Engine Cin/JohnBaumb Pass', desc: 'internal — applied the same architectural cleanups the download engine PR went through to the media server engine PR before review. (1) every server client (Plex / Jellyfin / Navidrome / SoulSync) now explicitly inherits `MediaServerClient` instead of relying on structural typing — drift in any class fails at the conformance test boundary. (2) generic accessors on the engine: `configured_clients()` (replaces per-server `if X and X.is_connected(): clients[name] = X` chains in web_server.py) and `reload_config(name=None)` (generic dispatch instead of per-client reload calls). (3) singleton factory: `get_media_server_engine()` / `set_media_server_engine()` matching the metadata + download engine shape. web_server.py boots via `set_media_server_engine(...)` so factory + global handle share state. (4) ~70 direct `plex_client.X` / `jellyfin_client.X` / `navidrome_client.X` / `soulsync_library_client.X` attribute reaches in web_server.py migrated to `media_server_engine.client(\'\').X`. ~60 standalone refs (truthy checks, media_client assignments, source-name tuples) also routed through the engine. (5) the per-server `plex_client` / `jellyfin_client` / `navidrome_client` / `soulsync_library_client` globals in web_server.py are gone entirely — engine owns the client instances now, every caller reaches via `media_server_engine.client(\'\')`. four multi-client consumers (`PlaylistSyncService`, `ListeningStatsWorker`, `WebScanManager`, discovery `SyncDeps`) refactored to take the engine instead of separate per-server kwargs. (6) `TrackInfo` and `PlaylistInfo` lifted out of `core/plex_client.py` / `jellyfin_client.py` / `navidrome_client.py` (each was defining a near-identical copy) into the neutral `core/media_server/types.py` module — same lift Cin caught on the download `TrackResult`/`AlbumResult`/`DownloadStatus` situation. consumers (matching engine, sync service) get one import. zero behavior change.' }, - { title: 'Internal: Media Server Engine Foundation', desc: 'internal — companion to the download engine refactor. introduces a media-server engine + plugin contract on top of the four server clients (plex / jellyfin / navidrome / soulsync standalone). pre-refactor web_server.py held four separate per-server client globals that every dispatch site reached individually. new `core/media_server/` package provides `MediaServerEngine` that owns the per-server clients + a small set of generic accessors (`client(name)`, `active_client()`, `configured_clients()`, `reload_config(name)`) so call sites use one canonical lookup pattern. plugin contract requires the four methods every server actually implements (is_connected, ensure_connection, get_all_artists, get_all_album_ids) — methods that exist on most-but-not-all servers (search_tracks, trigger_library_scan, get_library_stats, etc.) are listed in `KNOWN_PER_SERVER_METHODS` for discoverability and reached directly via `engine.client(name).` since there\'s no uniform safe-default that fits every method. honest scope: the four uniform-shape `is_connected` dispatches were lifted into `engine.is_connected()` (the one cross-server wrapper kept on the engine); the ~18 server-specific chains that do genuinely different per-server work (playlist track replace, metadata sync, scan strategies) stay explicit at the call site per the "lift what\'s truly shared" standard, but reach the per-server client through the engine. 42 tests pin: per-server observable behavior (4 server pinning files, 20 tests), engine surface + accessor contracts (15 tests), structural conformance + explicit-inheritance (9 tests). zero behavior change for users.' }, - { title: 'Drop SoundCloud Preview Snippets at Search Time', desc: 'soundcloud serves a ~30s preview clip for tracks that are gated behind go+ / login (very common for major-label uploads — official content basically doesn\'t exist on soundcloud, so what shows up is bootlegs, fan uploads, type beats, and these 30s previews). yt-dlp accepts the preview as the download payload, the post-download integrity check catches the duration mismatch and quarantines the file, but the user just sees "all candidates failed" with no explanation. previews also showed up in the candidate-review modal where clicking one bypassed validation and downloaded the same broken file. now `filter_soundcloud_previews` drops these candidates at every entry point — auto-search scoring, modal-cache fallback, AND the not-found raw-results path — so previews never reach the matcher OR the user. drops candidates < 35s or below half the expected duration, gated on expected being non-trivially long (>60s) so genuine short tracks still pass. also fixed a silent regression in the hybrid-fallback path where the per-source attribute removal left `getattr(orch, \'youtube\', None)` returning None for every source — fallback never fired. now resolves through the orchestrator\'s `client(name)` accessor.', page: 'downloads' }, - { title: 'Internal: Move Shared Download Dataclasses + Singleton Boot Path', desc: 'internal — two architectural cleanups on top of the download engine refactor. (1) `TrackResult`, `AlbumResult`, `DownloadStatus`, `SearchResult` lived in `core/soulseek_client.py` for historical reasons (they grew up there as the soulseek-only types and got exported when other download sources were added). every plugin imported these from the soulseek module just to satisfy the contract — coupling 8 clients to a sibling source for type imports only. moved them to `core/download_plugins/types.py` (the neutral plugin package) and updated all 14 import sites across deezer/hifi/lidarr/qobuz/soundcloud/tidal/youtube clients + the engine + matching engine + redownload + tests. clean break, no backward-compat re-export. (2) `web_server.py` now boots the orchestrator via `set_download_orchestrator(DownloadOrchestrator())` so the singleton factory + boot path share state — `get_download_orchestrator()` returns the same instance the global handle points at instead of lazily building a separate one. matches cin\'s `get_metadata_engine()` pattern.' }, - { title: 'Internal: Rename `soulseek_client` Global → `download_orchestrator`', desc: 'internal — followup cleanup. the global handle in web_server.py was named `soulseek_client` for historical reasons (the orchestrator was originally just the soulseek client and grew downstream sources around it), but the type has long been `DownloadOrchestrator` not `SoulseekClient`. renamed the global + every parameter/attribute that carried the legacy name across web_server.py, api/, core/downloads/*, core/search/*, core/streaming/*, services/sync_service.py, and the test fixtures (`MasterDeps.soulseek_client` → `download_orchestrator`, `init(soulseek_client_obj)` → `init(download_orchestrator_obj)`, etc). module path `core.soulseek_client` and class `SoulseekClient` (the actual soulseek-only client) are unchanged — only the orchestrator handle renamed. ~250 references touched, suite green.' }, - { title: 'Internal: Drop Backward-Compat Per-Source Attrs', desc: 'internal — followup to cin\'s download engine review. removed the `orchestrator.soulseek` / `.youtube` / `.tidal` / `.qobuz` / `.hifi` / `.deezer_dl` / `.lidarr` / `.soundcloud` attribute aliases that were preserved for backward compat. external callers (core/downloads/, core/search/, web_server.py) all migrated to the generic `orchestrator.client(\'\')` accessor — alias-aware (legacy `deezer_dl` resolves to canonical `deezer`), single source of truth via the registry. the orchestrator\'s own internal `self.soulseek` / `self.deezer_dl` reaches also routed through `client()` so the only place that knows about per-source identity is the registry. test fakes updated to expose `client(name)` instead of stuffing attributes; conformance test pinned to the new accessor contract. zero behavior change — just cleaner shape.' }, - { title: 'Internal: Download Engine Review Followup', desc: 'internal — three correctness fixes on top of the download engine refactor, all flagged in cin\'s pr review. (1) `engine.cancel_download(source_hint=\'deezer_dl\')` was silently routing deezer cancels to soulseek because the legacy alias never made it to the engine\'s plugin map — only the registry knew about it. fix: aliases now flow through `register_plugin` and `get_plugin` / `cancel_download` resolve them to the canonical name. (2) `_resolve_source_chain` filtered hybrid_order against canonical registry names only, so any user with `deezer_dl` in their config quietly dropped deezer from hybrid mode. fix: orchestrator normalizes through `registry.get_spec()` first. (3) the worker\'s terminal write was a read-then-write split — a cancel landing between the snapshot and the update could be overwritten back to errored / completed. fix: new atomic `update_record_unless_state` on the engine holds `state_lock` across the check + write; both `_mark_terminal` AND the success path use it now. also added generic `client(name)` / `configured_clients()` / `reload_instances(name?)` accessors on the orchestrator + a `get/set_download_orchestrator()` singleton matching cin\'s `get_metadata_engine()` shape, and migrated 30 external `soulseek_client.` reaches in web_server.py to `client("")`. 18 new tests pin every fix.' }, - { title: 'Internal: Typed Metadata Foundation', desc: 'internal — first step of a multi-pr migration to give the metadata pipeline a real contract. the codebase historically grew duck-typed extractors (`_extract_lookup_value(album_data, "id", "album_id", "collectionId", "release_id", default=...)`) at every consumer site because each provider returns its own response shape. ~150 of those across the codebase. new `core/metadata/types.py` defines canonical typed `Album` / `Track` / `Artist` dataclasses with strict required fields. per-source classmethod converters (from_spotify_dict, from_itunes_dict, from_deezer_dict, from_discogs_dict, from_musicbrainz_dict, from_hydrabase_dict) are the SINGLE place that knows each provider\'s wire shape. zero behavior changes in this pr — pure additive foundation. follow-up prs migrate consumers one at a time. full migration plan documented at docs/metadata-types-migration.md.', page: 'library' }, - { title: 'Internal: Migrate Album-Info Builders to Typed Path', desc: 'internal — steps 2+3 of the typed metadata migration in one pr. two album-info builders now route through `Album.from__dict()` when the caller passes a known source: `_build_album_info` (used by every album-tracks lookup) and the embedded album section of `_build_single_import_context_payload` (used by single-track import context resolution). legacy duck-typed extraction stays as the fallback when source is empty/unknown, raw input isn\'t a dict, or the typed converter raises — so a converter bug can\'t break album resolution or import context. caller-provided album_id / album_name / artist_name fallbacks apply on the typed path the same way they did on legacy. zero behavior change for existing callers since they don\'t pass a source yet — opt-in only. 22 new tests pin the typed path, the legacy fallback, and parametrized coverage across registered providers.' }, - { title: 'Internal: Migrate Discography + Quality Scanner to Typed Path', desc: 'internal — next round of the typed metadata migration. three more album-shape consumers now route through `Album.from__dict()` when the caller passes a known source: `_build_discography_release_dict` (artist discography release cards), `_build_artist_detail_release_card` (artist detail page release cards), and `_normalize_track_album` (quality scanner result normalization). legacy duck-typed extraction stays as the fallback when source is empty/unknown, raw input isn\'t a dict, or the typed converter raises — same safety contract as the prior migration steps. 20 new tests pin the typed path + legacy fallback + parametrized coverage across registered providers.' }, - { title: 'Fix: Maintenance Findings Badge Showed Inflated Count With Empty Findings Tab', desc: 'discord report (husoyo): duplicate detector and cover art filler badges showed "364 findings" / "31 findings" after a scan, but clicking into the findings tab showed nothing. cause: `_create_finding` silently dedup-skipped re-discovered issues (when an equivalent row already existed with status pending/resolved/dismissed) but the caller incremented `result.findings_created` regardless of whether a row was actually inserted. so on a re-scan that found the same problems as a prior scan, the badge snapshot recorded 364 even though zero NEW pending rows hit the db. fix: `_create_finding` now returns a bool (True on insert, False on dedup-skip / db error). all 16 repair jobs updated to only increment `findings_created` on True. new `findings_skipped_dedup` counter added to job results and surfaced in the scan log: "Done: 2791 scanned, 0 fixed, 0 findings (363 already existed), 0 errors" — so re-scans show a real count, and you can see at a glance how many findings were carried over from prior scans. also fixed a missing `job_id` kwarg in the album tag consistency job that was silently breaking finding creation for that scan. companion ux improvement: findings tab now auto-switches its status filter from "pending" to "all status" when 0 pending rows exist but resolved/dismissed/auto-fixed rows do — with a small notice so you can see what carried over instead of staring at an "all clear" empty state.', page: 'library' }, - { title: 'Internal: Download Source Plugin Contract', desc: 'internal — first step of a multi-step refactor on the multi-source download dispatcher. the orchestrator historically had 8 download sources (soulseek/youtube/tidal/qobuz/hifi/deezer/lidarr/soundcloud) hardcoded into 6+ different dispatch sites — `if username == "youtube" elif username == "tidal" elif ...` chains in `__init__`, search, download, get_all_downloads, cancel_download, etc. adding usenet (planned) would have meant 700+ lines of copy-paste across the same files. new `core/download_plugins/` package defines `DownloadSourcePlugin` (Protocol) — the canonical contract every source must satisfy: `is_configured`, `check_connection`, `search`, `download`, `get_all_downloads`, `get_download_status`, `cancel_download`, `clear_all_completed_downloads`. plus `DownloadPluginRegistry` — single source of truth for which sources exist, with name/alias resolution (legacy `deezer_dl` alias preserved). orchestrator now dispatches through the registry instead of hardcoded `[self.soulseek, self.youtube, ...]` lists. (note: this PR initially preserved `self.` attribute aliases for backward compat; followup commits in the same PR removed them — see the "Drop Backward-Compat Per-Source Attrs" entry above. external callers now reach individual clients via `orchestrator.client('')`.) zero behavior change for end users — pure additive foundation that lets future PRs extract shared logic (background thread workers, search query normalization, post-processing context) into the contract instead of copy-pasted across all 8 sources. 19 new tests pin every plugin class\'s structural conformance to the contract — drift in any source will fail at the test boundary instead of at runtime against a live download.' }, - { title: 'Internal: Download Engine — Background Worker, State, Fallback', desc: 'internal — followup to the download source plugin contract. lifts the duplicated thread-spawn boilerplate, per-source active_downloads dicts, and hybrid-fallback dispatch into a central `core/download_engine/` package. each streaming source (youtube, tidal, qobuz, hifi, deezer, soundcloud) used to hand-roll the same ~70 LOC of background thread management — semaphore-gated serialization, rate-limit sleep between downloads, state-dict updates for InProgress/Completed/Errored transitions, exception capture. ~490 LOC of copy-paste across 7 files. all of it gone now — `engine.worker.dispatch(source, target_id, impl_callable, ...)` owns thread spawning + semaphore + delay + state lifecycle. plugins provide only `_download_sync(download_id, target_id, display_name) → file_path`, the source-specific atomic download. per-source rate-limit policy declared via `RateLimitPolicy` (concurrency, delay) — engine reads at register time. cross-source state queries (`get_all_downloads`, `get_download_status`, `cancel_download`, `clear_all_completed_downloads`) read engine state directly instead of iterating per-source dicts. hybrid-mode search now goes through `engine.search_with_fallback(chain)` — same ordering / skip-unconfigured / swallow-per-source-exceptions semantics as before. every per-source migration commit gated by phase A pinning tests (54 tests across all 8 sources) so contract drift fails fast at the test boundary instead of at runtime against a live download. net: ~700 LOC removed across 6 client files, ~85 new engine + worker + rate-limit tests, suite green at every commit. zero behavior change for end users — same downloads, same lifecycle states, same hybrid mode. (per-source attribute aliases like `orchestrator.soulseek` were initially preserved here for backward compat; followup commits in the same PR cycle removed them — see the "Drop Backward-Compat Per-Source Attrs" entry above. soulseek-specific internals are now reached via `orchestrator.client(\'soulseek\')._make_request(...)`.) adding usenet now = one new client class + one registry entry, no orchestrator changes. follow-up: cin\'s metadata engine work may shape further refactors (e.g. extracting search retry / quality filter — left per-source for now since search code is genuinely 90% source-specific).' }, - { title: 'Discogs Collection in "Your Albums"', desc: 'discord request: pull your discogs collection into the your albums section on discover, similar to spotify liked albums. set your discogs personal access token on settings → connections (already there from prior work) and add discogs as one of the configured sources via the gear button on your albums. background fetcher pulls your full collection (all folders, all pages — capped at 5000 releases), normalizes artist names (strips discogs `(N)` disambiguation suffix), dedupes against any spotify/tidal/deezer-saved versions of the same album. clicking a discogs-only album opens with discogs context — full release detail (year, format, label, country, tracklist) from the /releases endpoint. clicking an album that exists in both your spotify saved AND discogs collection prefers spotify (download flow is more direct). discogs is physical-media-first so many releases won\'t have streaming equivalents — those still show in the grid but the modal flow may need to fall back to a name search to find a downloadable digital version.', page: 'discover' }, - { title: 'Drop Redundant "Your Spotify Library" Section on Discover', desc: 'discover page used to show two near-identical sections: "Your Albums" (cross-source aggregator across spotify/deezer/etc) AND "Your Spotify Library" (spotify-only). same UI, same grid, same filter / sort / download-missing controls — the spotify-only one was a strict subset of what your albums already covers. removed it. spotify saved albums still surface via the your albums section with spotify as one of its configured sources (gear button → configure sources). backend collection / storage is unchanged — the watchlist scanner still populates the spotify_library_albums cache for your albums to read.', page: 'discover' }, - { title: 'Library Disk Usage on Stats Page', desc: 'discord request (samuel [KC]): show how much disk space the library takes. new card on stats → system statistics shows total bytes + per-format breakdown (FLAC vs MP3 vs M4A bars). data comes from `tracks.file_size` populated during deep scan from whatever the media server already returns (plex MediaPart.size, jellyfin MediaSources[].Size, navidrome song.size, soulsync standalone os.path.getsize) — zero filesystem walk overhead. existing libraries see "Run a Deep Scan to populate" until the next deep scan fills in sizes; partial coverage shown as "X tracks measured (+Y pending)". migration is additive (NULL on legacy rows) so upgrading users have nothing to do.', page: 'stats' }, - { title: 'Fix: ReplayGain Wrote Same +52 dB Gain to Every Track', desc: 'noticed every downloaded track came out with `replaygain_track_gain: +52.00 dB` regardless of actual loudness. cause: parser used `re.search` which returned the FIRST `I:` (integrated loudness) reading from ffmpeg\'s ebur128 output. that\'s the per-window measurement at t=0.5s — almost always ~-70 LUFS because tracks start with silence/encoder padding. -18 (RG2 reference) - (-70) = +52 dB on every track. fix: parser now anchors to the `Summary:` block at the end of ffmpeg\'s output and reads the actual integrated loudness from there, not the silent-intro partial. defensive fallback uses the LAST per-window reading if Summary is missing (still better than the first). gains now reflect real per-track loudness.', page: 'downloads' }, - { title: 'Fix: Tracks Showed Completed When File Was Quarantined', desc: 'caught downloading kendrick mr morale: three tracks (rich interlude, savior interlude, savior) showed ✅ completed in the modal but were missing on disk. two layered bugs. (1) the post-process verification wrapper had a fallback that assumed success when no `_final_processed_path` was in context — but integrity-rejected files (which get quarantined instead of moved) leave that path unset, so the wrapper marked them complete. now wrapper explicitly checks `_integrity_failure_msg` and `_race_guard_failed` markers before the assume-success fallback. failed integrity = task marked failed, batch tracker notified with success=false. (2) acoustid skip-logic was too lenient — when fingerprint confidence was very high and either title OR artist matched a bit, it skipped verification with reason "likely same song in different language/script." that fired for english-vs-english by the same artist with the word "interlude" in both — same artist + 0.55 title sim = skip = wrong file accepted. tightened: skip now requires non-ASCII chars present (real language/script case) AND artist match, OR very high title similarity (≥0.80) AND artist match. english-vs-english with very different titles by same artist no longer skipped — verification correctly returns FAIL and the wrong file gets quarantined.', page: 'downloads' }, - { title: 'Stop Navidrome From Splitting Albums Over Inconsistent MBIDs', desc: 'discord report (samuel [KC]): tracks of the same album sometimes carry different MUSICBRAINZ_ALBUMID tags, which causes navidrome to split the album into multiple entries. two-part fix: (1) the MBID Mismatch Detector now does a second scan that groups tracks by db album, finds the consensus (most-common) album mbid, and flags dissenters — fix action rewrites the dissenter\'s tag to match. catches existing inconsistencies in your library. (2) root cause: per-track musicbrainz release lookups went through an in-memory cache that\'s capped at 4096 entries and dies on server restart, so big libraries / restarts could resolve different release ids for tracks of the same album. added a persistent sqlite-backed cache so a release mbid resolved ONCE for an album applies to every future track of that album for the install\'s lifetime. strictly additive: any failure in the persistent layer falls through to the live musicbrainz lookup exactly as before.', page: 'library' }, - { title: 'Lidarr: Right Track Lands on Disk + Profile Lookup Stops Failing', desc: 'lidarr is an album-grabber — when you ask for one track it grabs the whole album, then we pick the wanted track out. old code blindly took the first imported file as the result, so any track you asked for got mistagged as track 1 of the album. now matches the wanted title against lidarr\'s track list (with punctuation-tolerant fuzzy compare) and copies only that file. also fixed a hardcoded `metadataProfileId=1` that broke artist-add on installs where someone had renamed/recreated profiles, and a polling-loop bug where the inner break never escaped the outer poll loop so completion detection was delayed. settings tooltip updated to be honest: lidarr is best for full-album grabs and effectively a no-op for playlist sync (track searches return nothing useful, hybrid mode falls through to your other sources).', page: 'settings' }, - { title: 'SoundCloud as a Download Source', desc: 'discord request (toasti): some tracks (DJ mixes, sets, removed-from-spotify exclusives) only live on soundcloud. soundcloud now plugs into the existing download-source picker on settings → downloads — pick "SoundCloud Only" or include it in the hybrid order alongside soulseek / youtube / tidal / qobuz / hifi / deezer / lidarr. anonymous-only (no account needed); quality is whatever soundcloud serves anonymously, typically 128 kbps mp3 or aac depending on the upload. soundcloud doesn\'t expose lossless to anyone, so don\'t expect flac. follows the exact same wiring contract as every other download source — search dispatch, hybrid fallback, queue / cancel / clear, sidebar source label, provenance + library history all work plug-and-play.', page: 'settings' }, - { title: 'Fix Qobuz Connection Not Sticking After Login', desc: 'logging in via the qobuz connect button on settings showed "connected: (active)" but underneath an error said "qobuz not authenticated...", and the dashboard indicator stayed yellow. cause: two separate qobuz client instances run side by side (one for the auth flow, one for the enrichment worker) and login only updated the first one. now the worker\'s client gets synced from config the moment login / token / logout completes, so the dashboard indicator goes green and connection-test stops yelling.', page: 'settings' }, - { title: 'Fix Lossy Copy Not Deleting Original FLAC', desc: 'with lossy copy enabled and "delete original" turned on (you wanted an mp3-only library), every download still left both the flac and the converted mp3 sitting in the same folder. the setting was being read but never acted on during the conversion step. now the original gets removed right after a successful conversion, with a same-path safety check + graceful handling if the original is already gone or locked.', page: 'settings' }, - { title: 'Watchlist Stops Re-Downloading Tracks That Already Exist', desc: 'a track that was already on disk got re-downloaded by the watchlist on every scan because the library had stale album metadata for it (file tagged on the wrong album by an old import) and the album fuzzy comparison declared the track missing. now the watchlist also matches by stable external IDs (spotify / itunes / deezer / tidal / qobuz / musicbrainz / audiodb / hydrabase / isrc) before falling through to the fuzzy block — so any track whose tags or DB row carry a matching ID is recognized as already present regardless of album drift. provider-neutral, falls through to existing fuzzy logic for older imports without IDs.', page: 'watchlist' }, - { title: 'Persist Source IDs at Download Time + Backfill on Sync', desc: 'every download already collects spotify/itunes/deezer/tidal/qobuz/musicbrainz/audiodb/hydrabase/isrc IDs during post-processing, but for plex/jellyfin/navidrome users they got dropped on the floor — only enrichment workers eventually wrote them onto the tracks row, hours later. now those IDs persist to the track_downloads table immediately, the media-server sync code copies them onto the new tracks row the moment it gets created, and the watchlist scanner has a second-tier fallback to query provenance directly when the tracks row hasn\'t been synced yet. closes the enrichment-wait window — freshly downloaded files are recognizable on the very next watchlist scan instead of after enrichment catches up.', page: 'library' }, - { title: 'Fix Tidal Auth Error 1002 for Docker / Remote Access', desc: 'tidal returned error 1002 ("invalid redirect URI") on every authentication attempt for users accessing soulsync from a network IP. cause: when the redirect_uri config field was empty (which it usually was, because the UI just shows the default as a placeholder without saving it), the /auth/tidal route silently overrode the constructor default with a uri built from request.host — http://192.168.x.x:8889/tidal/callback. that didn\'t match what users had registered in their tidal developer portal (http://127.0.0.1:8889/tidal/callback per the docs and UI default), so tidal rejected the authorize request before users ever saw the consent screen. fix: drop the request-host fallback entirely. empty config now falls back to the constructor default that matches the documented portal registration. the existing post-auth swap-step instructions handle the docker/remote-access case as designed.', page: 'settings' }, - { title: 'Auto-Import: Live Per-Track Progress in History', desc: 'dropping an album into the staging folder used to leave the auto-import history blank for the entire processing window — sometimes 5+ minutes for a full album — because the database row only got written after every track was post-processed. now an in-progress row gets inserted up-front (status=processing) the moment processing starts, then updated to completed/failed when done. the status indicator + progress bar show "processing speak now — track 3/14: mine", and the history card itself gets a pulsing "Processing" badge, swaps its meta line to "track 3/14: mine", and highlights the currently-processing row in the expanded track list (with prior tracks dimmed as done). one row per album, not per track, so the history list stays clean.', page: 'import' }, - { title: 'Reject Broken Files from slskd Before Tagging', desc: 'slskd sometimes reports a download as complete when the file is actually broken — truncated transfer, corrupted FLAC frames, or the wrong file matched on a similar filename. those slipped through into the library and surfaced as "song plays for 5 seconds and stops" or "track shows the wrong duration in plex." now every download gets a fast integrity check after the file stabilizes but before tagging / library sync: file size sanity (catches 0-byte and stub transfers), mutagen parse (catches header damage and wrong-format-with-right-extension cases), and duration agreement against the metadata source\'s expected length within a 3-second tolerance (5s for tracks over 10 minutes). failed files get quarantined to `ss_quarantine/` with a JSON sidecar explaining the failure, and the download slot is freed so a retry from another candidate can run.', page: 'downloads' }, - { title: 'Auto-Import: Multi-Disc Albums + Featured-Artist Tag Handling', desc: 'two longstanding auto-import gaps that surfaced when a kendrick lamar deluxe rip got dropped into staging. (1) folders containing only `Disc 1/`, `Disc 2/` subfolders (no loose audio at the parent level) used to be invisible to the scanner — disc folders were only attached to a parent when the parent had its own loose tracks. now scanner treats a parent of disc-only subfolders as the album candidate. (2) tag identification grouped files by `(album, artist)` — but per-track artist often varies on albums with features ("kendrick lamar" vs "kendrick lamar, drake" vs "kendrick lamar, dr. dre"), which fragmented the consensus and rejected real albums. now groups by album first, picks the dominant artist within that album group; also prefers `albumartist` tag over per-track `artist` since the former is the album-level identity. as a defensive bonus, when the staging folder itself becomes the candidate (raw disc folders dropped at the root with no album wrapper), the folder-name fallback gets skipped — the name "Staging" was matching against random albums in the metadata source.', page: 'import' }, - { title: 'Album Completeness Auto-Fill Works on Docker / Shared Library Setups', desc: 'github issue #476 (gabistek): the "auto-fill" button on the album completeness findings page returned `Could not determine album folder from existing tracks` for every album on docker setups (and any setup where the media-server library lives somewhere other than the soulsync transfer/download folders). cause: the repair worker\'s path resolver only probed the transfer + download folders, ignoring the user-configured `library.music_paths` and the plex-reported library locations. that missing search space meant docker users — whose plex/jellyfin library is bind-mounted at `/music` while soulsync\'s transfer is at `/transfer` — got silent "file not found" results for every existing track. extracted the full resolver (with library + plex sources) into a shared `core/library/path_resolver.py` and wired it into all five repair-worker call paths plus the four jobs that had their own incomplete copy. side benefit: every other repair job (dead file cleaner, mbid mismatch detector, lossy converter, acoustid scanner, unknown artist fixer) also stops missing files in the media-server library mount.', page: 'library' }, - { title: 'Sidebar Library Button Shows Artist Breadcrumb', desc: 'when you open an artist detail page (from library, search, or the global search popover), the sidebar Library button now lights up and rewrites its label to "Library / Artist Name" — long names truncate with an ellipsis and the full name shows on hover. revertes to plain "Library" when you leave. purely visual, no functionality change.', page: 'library' }, - { title: 'Enrichment Bubble Routes Consolidated', desc: 'internal — every dashboard enrichment bubble (musicbrainz, spotify, itunes, deezer, discogs, audiodb, lastfm, genius, tidal, qobuz) used to hit its own per-service status / pause / resume route in web_server.py. unified them under a single registry-driven endpoint set: /api/enrichment//. spotify\'s rate-limit guard, lastfm/genius yield-override behavior, and tidal/qobuz extra status fields are encoded as data on the registry. 27 new tests cover the registry behavior.' }, - { title: 'Drop Old Per-Service Enrichment Routes', desc: 'internal — followup to the registry consolidation. now that the dashboard has cut over to /api/enrichment//, deleted the 30 hand-rolled per-service routes from web_server.py (musicbrainz/audiodb/discogs/deezer/spotify/itunes/lastfm/genius/tidal/qobuz status+pause+resume). ~510 lines gone from the monolith, no behavior change.' }, + '2.5.7': [ + { date: 'May 19, 2026 — 2.5.7 release' }, + { title: 'Fix: MusicBrainz manual search missing results', desc: 'the Fix popup and manual library service search were using strict Lucene phrase-match queries against the `recording` / `release` / `artist` fields — diacritics ("Bjork" vs canonical "Björk"), bracketed suffixes like "(Live)", and any AND-clause mismatch all killed recall. switched user-facing manual lookups to bare queries that hit MB\'s alias / sortname indexes with diacritic folding. enrichment workers stay strict for precision.' }, + { title: 'Fix: MusicBrainz album clicks 404ing in enhanced search', desc: 'every click on a MusicBrainz album result was silently 404-ing — the /release fetch was passing `cover-art-archive` as an `inc` param, which MB rejects with 400 (that field is returned on every release response by default, no include needed). dropped the bad include; album detail now loads correctly.' }, + { title: 'Fix popup: paste a MusicBrainz URL or MBID to match directly', desc: 'new escape hatch on the Fix Track Match modal (the 🔧 Fix button on mirrored / YouTube / Tidal / Deezer / Beatport / ListenBrainz / Spotify-public discovery rows). when fuzzy search keeps ranking the wrong recording among many same-title versions, paste the MusicBrainz recording URL like `https://musicbrainz.org/recording/` or the bare UUID into the new field and hit "Look up". skips all fuzzy logic, resolves straight to that record, and runs it through the same confirm + match pipeline.' }, + { title: 'Fix popup: MusicBrainz added to the auto-search cascade', desc: 'the Fix Track Match modal used to query only Spotify → Deezer → iTunes for the auto-search, leaving MusicBrainz out of the loop entirely — even for users with MusicBrainz set as their primary metadata source. now MB is part of the cascade. when MB is your primary, it gets queried first; otherwise it sits as the last fallback. catches niche / non-mainstream / canonical-with-diacritics recordings that the commercial sources miss. Discogs is intentionally absent — Discogs has no track-level search API.' }, + { title: 'Fix: Docker basic-search streaming silently failed under rootless Docker', desc: 'the streaming "Play" flow on the basic search page tried to create `/app/Stream` lazily at runtime, which fails silently when the container runs under rootless Docker / Podman (in-container root can\'t write to `/app`). pre-baked the directory at image build time, matching the same pattern that fixed `/app/Staging` earlier in the cycle. non-persistent — no volume needed.' }, + { title: 'Fix: slskd-unreachable log spam during non-Soulseek downloads', desc: 'when slskd was configured but not actually running (or unreachable on its configured port), the `/api/downloads/status` polling loop fanned out to every download plugin including Soulseek, producing one `ERROR - Cannot connect to host ... [Name or service not known]` log line per poll for the entire duration of any download — visible spam even when the user wasn\'t using Soulseek at all. Connection failures now emit one WARNING with actionable context (start slskd, or clear the slskd_url if you don\'t use Soulseek) and demote subsequent failures to debug. The flag resets on the next successful slskd response so a later outage warns again.' }, + { title: 'Fix: MusicBrainz "Other" release-groups now visible in discography', desc: 'MB tags music videos, one-off web releases, and broadcast singles with `primary-type=Other` — common pattern for Vocaloid producers, JP indie artists, and some Western indie acts. The release-group browse filter only requested `album|ep|single`, dropping every Other-typed group at the API layer. Combined with the inline type mapper defaulting unknown primary types to "album", this hid legitimate tracks from the artist detail page and left downloaded tracks orphaned (visible in track counts but not bound to any visible album / single card). Centralised the type-mapper into one shared helper (`core/metadata/release_type.py`) used by every provider\'s raw→Album projection, added `Other` and `Broadcast` handling that routes those release-groups into the Singles section, and added `other` to the MB API filter. For inabakumori, this surfaces 5 previously-invisible releases. 23 unit tests pin the mapper contract; existing 65 search-adapter tests still green.' }, + { title: 'Fix: quarantined files no longer re-downloaded on auto-wishlist cycles', desc: 'when a file failed AcoustID verification and got quarantined, the next auto-wishlist run would search for the same track again, and the candidate picker\'s deterministic quality ranking kept selecting the same `(uploader, filename)` source — re-downloading and re-quarantining the exact same file every cycle. Users woke up to hundreds of duplicate `.quarantined` entries from a single bad upload. The Soulseek candidate filter now reads quarantine sidecars and drops any candidate whose `(username, filename)` matches a previously-quarantined source before the quality picker ranks it. Filesystem read on every search (sub-ms in practice). Approving or deleting a quarantine entry removes its source from the dedup set automatically — the user can give the source a second chance by explicitly approving / deleting the quarantine record.' }, + { title: 'Fix: Unknown Artist Fixer tool crashed on every run', desc: 'the "Fix Unknown Artists" repair job crashed instantly with `ImportError: cannot import name \'_build_path_from_template\'` — totally unrunnable. Commit ca5c9316 ("Rewrite Library Reorganize job to delegate to per-album planner") moved the private path-builder + quality-string helpers out of `core.repair_jobs.library_reorganize` and into the import pipeline (`core.imports.paths` / `core.imports.file_ops`), but the Unknown Artist Fixer\'s deferred-import path was left pointing at the old module. Re-wired to the new locations. Added two regression tests that exercise the deferred imports directly — the next refactor that moves these helpers fails CI rather than reaching the user.' }, ], - '2.4.1': [ - // --- May 1, 2026 — patch release --- - { date: 'May 1, 2026 — 2.4.1 release' }, - - // --- Watchlist / wishlist correctness --- - { title: 'Watchlist No Longer Re-Downloads Compilation Tracks', desc: 'spotify and your media server name compilation albums differently — "napoleon dynamite (music from the motion picture)" vs "napoleon dynamite ost". the watchlist scanner used a strict 0.85 fuzzy threshold against the raw names, which always failed for soundtracks / deluxe-editions, so it kept re-adding the same track to the wishlist on every scan. one user reported the same song downloaded 7 times. now strips qualifier parentheticals (music from..., ost, deluxe edition, remastered) before comparing, with a volume / disc / part guard so vol 1 vs vol 2 still count as different.', page: 'watchlist' }, - { title: 'Duplicate Detector Catches slskd Dedup Orphans', desc: 'when a track downloaded multiple times, slskd appended "_" to each copy and the media-server scan often parsed inconsistent titles for them — so the duplicate detector\'s title-bucket pass never compared them. added a second pass that re-buckets leftover tracks by canonical filename stem (slskd dedup tail stripped). seven copies of the same song in one folder now get caught as one duplicate group.', page: 'library' }, - { title: 'Clean Up slskd Dedup Orphans After Import', desc: 'slskd appends "_" to a download when the destination file already exists (retried partials, the same track in multiple playlists, etc.). the canonical file imported fine but the timestamp-suffixed siblings sat in the downloads folder forever. now they get pruned right after each successful import.', page: 'downloads' }, - { title: 'Bulk Watchlist Add: Try Every Source ID Before Failing', desc: 'bulk-add to watchlist used to give up if your active metadata source didn\'t resolve the artist. now falls back through every cached source id (spotify, deezer, itunes, discogs, hydrabase) before declaring failure. fixes adds going dead when one source rate-limited.', page: 'watchlist' }, - { title: 'Wishlist Respects Configured Providers', desc: 'wishlist UI was hardcoded to spotify in some places — labels, retry copy, source defaults. now mirrors your active primary metadata source so deezer / itunes / discogs / hydrabase users see consistent text everywhere.', page: 'sync' }, - { title: 'Quality Scanner Respects Primary Metadata Provider', desc: 'quality scanner queried spotify regardless of your configured primary source, leaking spotify api calls and ignoring discogs/hydrabase data. refactored to honor the primary provider for matching; artwork preserved on wishlist handoff.', page: 'library' }, - { title: 'Wishlist Track Counts Coerced Before Category Checks', desc: 'wishlist could crash on category gating when a track count came back as a string from one source vs an int from another. now coerces to int before checking single / EP / album thresholds.' }, - - // --- Match engine correctness --- - { title: 'Featured-Artist Tracks Match Across Discography Completion', desc: 'tracks where the watched artist is a feature (not the primary) used to be treated as missing during discography completion checks. now matches against the per-track artist list so guest spots count.', page: 'library' }, - { title: 'Soundtrack Tracks Match Against Per-Track Artist', desc: 'OST/compilation tracks were matched against the album\'s primary artist (often "Various Artists") instead of the actual track artist. fixed — soundtrack tracks now match against the track\'s real artist credit, and the dead fallback path that used to swallow the miss is gone.', page: 'library' }, - - // --- Spotify auth flow rework (kettui PR) --- - { title: 'Spotify Auth Flow: Clearer UI + Reliable Sync', desc: 'rewrote the spotify connection flow on settings → connections. separated "needs auth" / "connecting" / "connected" states with explicit labels, fixed completion-sync races where the page would say connected before the token finished saving, and surfaces auth-completion failures as toasts instead of silent fails. service status reads are simpler and more honest about state.', page: 'settings' }, - { title: 'Spotify Worker Pauses on Non-Spotify Primary', desc: 'spotify enrichment worker kept running and burning api budget even when spotify wasn\'t your primary source. now pauses unless spotify is selected. also cut the per-day budget cap from a higher value to 500 calls so accidental quota burns are bounded.', page: 'dashboard' }, - { title: 'Tidal Auth Instructions Show Tidal\'s Callback Port', desc: 'tidal connect screen displayed spotify\'s callback port number in its setup steps. fixed to show tidal\'s actual port so the redirect URI users set up actually works.', page: 'settings' }, - - // --- Discogs --- - { title: 'Discogs Primary Source Gated by Token', desc: 'selecting discogs as your primary metadata source without a token now reverts gracefully instead of erroring on every call. token presence is the gate — set it on settings → connections to enable discogs as primary.', page: 'settings' }, - - // --- Imports --- - { title: 'Parallel Singles Import (3 Workers)', desc: 'singles / EP imports used to process serially. now run through a 3-worker thread pool so a long backlog of liked-songs imports finishes ~3x faster. also routes singles + EPs through the album_path template so they file correctly.', page: 'sync' }, - - // --- Duplicate detector --- - { title: 'Same-Physical-File Duplicates No Longer Flagged', desc: 'if you bind-mount the same music folder into both soulsync (e.g. /app/Transfer) and plex (e.g. /media/Music), each row in the DB pointed at the same file via a different mount root and showed up as a "duplicate". detector now recognizes this — same trailing path segments + matching durations + different mount roots = filtered out.', page: 'library' }, - - // --- Bug fixes --- - { title: 'Fix Config DB Lock Spam on Slow Disks (#434)', desc: 'on slow / heavily-loaded disks, sqlite settings DB writes raced and spammed the log with "database is locked" errors every few seconds. added a retry loop with exponential backoff and bounded retry count. silent on healthy systems, recovers on slow ones.', page: 'settings' }, - { title: 'Fix Bulk Discography Losing Album Source Context (#399)', desc: 'bulk discography downloads weren\'t carrying the album\'s source provider through the pipeline, so downstream lookups defaulted to the wrong source. fixed by threading source context through every step.', page: 'sync' }, - { title: 'Beatport Tab Hidden Temporarily', desc: 'beatport rolled out cloudflare turnstile on every public page, so the scraper that powered the beatport tab now hits a bot challenge instead of html. their official oauth api is locked behind partner registration that isn\'t open to the public. hid the tab on sync until we find a workaround — backend endpoints are kept in code so revival is a one-line html change.', page: 'sync' }, - { title: 'Surface Handler-Returned Errors in Automation last_error', desc: 'automation actions could return an error string but the engine swallowed it — last_error stayed blank, debugging was painful. now propagates returned errors into last_error so you can see what failed and why.', page: 'stats' }, - { title: 'Silence Shutdown-Time Logger Noise in CI', desc: 'pytest closes log handles before atexit runs — produced "I/O operation on closed file" stack traces in CI stderr on every test run. registered a final atexit handler that toggles logging.raiseExceptions off so shutdown is silent.' }, - - // --- Performance / infra --- - { title: 'Service Worker for Cover Art + Installable PWA', desc: 'cover art used to re-fetch from the CDN on every library / discover page visit. now a service worker caches images locally — second visit serves art instantly from disk, no network hit. also added a PWA manifest so soulsync can be installed to home screen / desktop as a standalone app (chrome / edge / safari → install soulsync). cache versioned so future strategy changes invalidate cleanly.' }, - { title: 'Browser Caching for Static Assets + Discover Pages', desc: 'static assets (js / css / icons) now get a 1-year browser cache instead of revalidating on every page load. safe because the existing ?v=static_v cache-bust query changes every server restart, so deploys still ship live. discover pages (hero, similar artists, recent releases, deep cuts) now cache 5 minutes browser-side so toggling between sections doesn\'t re-fetch everything.', page: 'discover' }, - { title: 'Faster Docker Startup — yt-dlp Pinned', desc: 'docker startup used to run `pip install -U yt-dlp` on every container start. removed that — yt-dlp is now pinned in requirements.txt so startup is fast and reproducible. tradeoff: youtube fixes ship via soulsync releases now instead of next container restart.' }, - - // --- Security --- - { title: 'Lock Down Socket.IO CORS', desc: 'socket.io was accepting websocket connections from any origin (cors=*). now defaults to same-origin only. if your websocket fails after updating, the server logs a clear warning with the rejected origin — add it to settings → security → allowed websocket origins.', page: 'settings' }, - { title: 'Settings Endpoints: Admin-Only', desc: 'the /api/settings endpoints (read, write, log-level, config-status, verify) had no auth gate — any logged-in profile could read or change service tokens, oauth secrets, api keys. now admin-only. single-admin setups (no multi-profile config) work transparently as before.', page: 'settings' }, - - // --- Internal / refactoring --- - { title: 'Major web_server.py Decomposition', desc: 'internal — pulled ~30 routes / workers / helpers out of web_server.py into focused modules under core/ (search, automation, stats, discovery, library, downloads, workers, artists, connection, debug, watchlist auto-scan, retag, redownload, library service search, duplicate cleaner, monitor, validation, staging, etc.). meaningfully smaller monolith, better unit-testable seams, no behavior change.' }, - { title: 'Metadata Helpers Reorganized into Packages', desc: 'internal — metadata helpers and runtime client management moved into proper packages (core/metadata/, core/imports/), with profile spotify cache living in the registry. clearer ownership, fewer cross-module reach-ins.' }, - { title: 'Stats Endpoints Lifted to core/stats', desc: 'internal — moved /api/stats/* and /api/listening-stats/* logic out of web_server.py into core/stats/queries.py with full test coverage.' }, - { title: 'Search Endpoints Lifted to core/search', desc: 'internal — moved /api/search and /api/enhanced-search/* logic into core/search/ (cache, sources, library_check, stream, basic, orchestrator). 612 fewer lines in web_server.py, 94 new tests.' }, - { title: 'Automation Endpoints Lifted to core/automation', desc: 'internal — moved /api/automations/* CRUD + run + history routes, progress tracking helpers, and signal collection into core/automation/ (api, progress, signals). 383 fewer lines in web_server.py, 72 new tests.' }, + '2.5.6': [ + { date: 'May 18, 2026 — 2.5.6 release' }, + { title: 'MusicBrainz as Primary Metadata Source', desc: 'MusicBrainz is now a full primary metadata source on equal footing with Deezer, iTunes, Spotify, and Discogs. switch to it in Settings → Metadata Source — always available, no account or API key needed, rate-limited to 1 req/sec. covers all primary source flows: search, album/track/artist lookup, watchlist scans, discover hero, similar artist backfill, artist map.', page: 'settings' }, + { title: 'Fix: MusicBrainz artist detail showing MBID as name', desc: 'clicking a MusicBrainz artist from search results was showing the raw MBID as the artist name on the detail page. URL-driven routing (PR #644) no longer passes the display name to the backend, so the source detail endpoint now looks it up directly from MusicBrainz by MBID.' }, + { title: 'Fix: artist detail back button always showing "← Back"', desc: 'PR #644 removed the back-button label logic along with the origin stack. restored: a label stack (separate from browser history, which handles actual navigation) tracks where you came from across the full similar-artist chain — "← Back to Search", "← Back to Artist A", "← Back to Artist B", etc. API response backfills the current artist name so the stack has real names when clicking similar artists.' }, + { title: 'Fix: Amazon search albums/artists missing, album downloads all track 01', desc: 't2tunes proxies Amazon Music and uses 400 to signal transient failures — first API call in a session hit this consistently, so album/artist searches always failed while track search (called 0.5s later) scraped through. added up to 3 retries with backoff on t2tunes-specific 400s. also: all search methods were using types=track,album but t2tunes album-type queries are broken — switched everything to types=track and derive albums/artists from track metadata instead. track numbers from album downloads were also always 1 — added index-based fallback when t2tunes tags omit trackNumber.' }, ], - '2.4.0': [ - // --- April 26, 2026 — Search & Artists unification + reorganize queue --- - { date: 'April 26, 2026 — 2.4.0 release' }, - { title: 'Reorganize Queue Polish', desc: 'cleaned up some race conditions in the reorganize queue. cancel + bulk dedupe behavior is solid now. preview button no longer gets stuck disabled on errors.', page: 'library' }, - { title: 'Reorganize Queue with Live Status Panel', desc: 'reorganize is now a queue with a live status panel. spam-click all you want — items run one at a time and you can keep browsing while they go. expand the panel to see queue + cancel buttons.', page: 'library' }, - { title: 'Album Completeness Job Actually Works', desc: 'completeness job was finding zero issues for everyone. now it works — uses real expected track counts from your metadata source instead of comparing your library to itself.', page: 'library' }, - { title: 'Reorganize Routes Through the Download Pipeline', desc: 'reorganize now uses the same pipeline downloads use. fixes 3-disc albums collapsing to single-disc and tracks silently disappearing on you. extracted to core/library_reorganize.py.', page: 'library' }, - { title: 'Spotify: Longer Post-Ban Cooldown', desc: 'bumped the post-ban cooldown from 5 to 30 minutes. first call after a ban was getting re-banned within seconds because spotify\'s memory outlasts the cooldown.', page: 'dashboard' }, - { title: 'Tidal: No More Silent Quality Downgrades', desc: 'tidal was silently serving 320kbps when you asked for hires. now it rejects the downgrade and the fallback chain advances properly — or fails honestly if you have "hires only, no fallback" set.', page: 'downloads' }, - { title: 'Search Source Picker Icon Row', desc: 'search page now has a row of source icons above the bar — one per source. typing only searches the active source instead of fanning out to all of them. click another icon to switch.', page: 'search' }, - { title: 'Per-Query Source Cache', desc: 'switching back to a source you already searched is instant — results are cached for the current query. cache resets when you type a new query. ~6-7x fewer api calls per search.', page: 'search' }, - { title: 'Global Search Widget Source Parity', desc: 'the sidebar global search popover got the same source icon row + cache dots + fallback banner as the full search page.', page: 'search' }, - { title: 'Rate-Limit Fallback Banner', desc: 'if the backend swaps your selected source for a working one (e.g. spotify rate-limited → deezer), you get a small amber banner explaining the swap. icon for the failed source gets an amber border.', page: 'search' }, - { title: 'Explicit Source Selection on /api/enhanced-search', desc: 'enhanced-search endpoint takes a source param now to skip the fan-out backend-side. cache keys isolate per-source so single and multi-source results don\'t collide.', page: 'search' }, - { title: 'Shared Enhanced-Search Fetch Helper', desc: 'internal — search dropdown and global widget share one fetch helper now instead of duplicating the post boilerplate.', page: 'search' }, - { title: 'Search Page Renamed to /search', desc: 'search page is now /search instead of the confusing /downloads (which clashed with the actual downloads page). old urls still work.', page: 'search' }, - { title: 'Embedded Download Manager Removed from Search Page', desc: 'killed the duplicate download manager on the search page (~330 lines of dead code). dedicated downloads page is the only one now.', page: 'search' }, - { title: 'Artists Sidebar Entry Retired', desc: 'removed the artists sidebar entry — unified search already does what it did. old /artists urls still resolve.', page: 'search' }, - { title: 'Artist Detail Back Button Fallback', desc: 'back button on inline artist detail uses browser history when you arrived from outside the artists page, instead of dumping you on an empty artists search.', page: 'search' }, - { title: 'Interactive Help Updated for Unified Search', desc: 'rewrote the click-for-help annotations and the first-download tour for the new search page. retired the standalone browse-artists tour.', page: 'help' }, - { title: 'Unified Source-Picker Controller', desc: 'internal — search page and global widget share one controller now (~380 lines of duplicate state/fetch/render code gone). bug fixes land everywhere at once.', page: 'search' }, - { title: 'Fix Clean Search History Automation Crashing', desc: 'hourly clean-search-history automation was crashing on a stale base_url path. fixed.', page: 'stats' }, - { title: 'Search Results Always Visible', desc: 'killed the show/hide results toggle. visibility is just based on whether you\'ve typed a query.', page: 'search' }, - { title: 'Cached Search Results Restore on Navigate-Back', desc: 'leaving and coming back to /search now re-renders your last query\'s results from cache instead of hiding them.', page: 'search' }, - { title: 'Fix Soulseek Handoff from Global Search', desc: 'clicking soulseek in the global search popover used to run metadata search against your default source instead of basic file search. fixed.', page: 'search' }, - { title: 'Stale Search Requests No Longer Flash Empty', desc: 'fast retypes used to flash an empty state for a moment while the new fetch was still mid-flight. added a request-sequence token so old responses don\'t clobber new ones.', page: 'search' }, - { title: 'Soulseek Icon Dims When slskd Isn\'t Configured', desc: 'soulseek icon dims if you don\'t have slskd set up. clicking it routes to settings → downloads instead of failing silently.', page: 'search' }, - { title: 'Fix Discover Hero View Discography 404', desc: 'view discography on the discover hero was 404ing for non-library artists. fixed by passing the source through to /api/artist-detail.', page: 'discover' }, - { title: 'MusicBrainz Search Actually Works', desc: 'musicbrainz search was returning empty/garbage results and taking 30+ seconds. rewrote it — artist, track, and album searches all work now and complete in ~3 seconds on cold cache.', page: 'search' }, - { title: 'MusicBrainz Search Follow-Ups', desc: 'three more musicbrainz fixes — artist images now resolve via itunes/deezer fallback, total_tracks off-by-one fixed, and "artist title" queries no longer browse the whole discography.', page: 'search' }, - ], - '2.39': [ - // --- April 22, 2026 --- - { date: 'April 22, 2026' }, - { title: 'Fix Wrong-Artist Tracks Silently Downloading from Tidal', desc: 'A user reported that searching for "Leave A Light On" by Maduk on Tidal silently downloaded Tom Walker\'s (completely different) song of the same name, embedding Maduk metadata into Tom Walker\'s audio. Two layers of defense were failing: (1) the candidate artist gate used `< 0.4` similarity and "maduk" vs "tom walker" scored exactly 0.400, slipping past the fencepost — raised to `< 0.5`. (2) AcoustID verification correctly identified the mismatch but returned SKIP (accept) instead of FAIL (quarantine) when title matched but artist was clearly different and the expected artist was absent from every recording. Now returns FAIL when artist similarity < 0.3 (clear mismatch); preserves SKIP for the ambiguous 0.3-0.6 range (covers/collabs/formatting differences)', page: 'sync' }, - { title: 'Tidal Search Falls Back to Shortened Queries on 0 Results', desc: 'Tidal\'s search chokes on long queries with multiple qualifier words (e.g., "maduk transformations remixed fire away fred v remix" returns nothing, but dropping "fred v remix" works). Search now retries with up to 4 progressively-shortened variants when the original returns 0 results. Qualifier-safe: if the original query mentions Live/Remix/Acoustic/etc., fallback results must still contain those keywords in their track names — otherwise a shortened query could silently downgrade "(Live)" to the studio version. Returns ([], []) if no variant preserves the qualifiers, same as before', page: 'sync' }, - ], - '2.38': [ - // --- April 21, 2026 (late) --- - { date: 'April 21, 2026 (late)' }, - { title: 'Fix Missing Cover Art on Manually Fixed Discovery Tracks', desc: 'The cache matched_data built by the fix modal dropped the image_url and album.images fields when album came back as a bare string (common for Deezer/iTunes search results). Result: re-discovery used the cached match but downloads showed no artwork. Cache writes now carry image_url through to album.images + top-level matched_data, matching what the in-memory state already did. Re-fix the track to refresh its cache entry (INSERT OR REPLACE)', page: 'sync' }, - { title: 'Fix Manual Discovery Fixes Lost After Restart (Non-Spotify Users)', desc: 'When you clicked Fix on a discovery track and picked a manual match, the cache save hardcoded the provider as "spotify" regardless of your configured primary metadata source. On re-scan, the worker queried the cache with your actual primary (Deezer, iTunes, Discogs, Hydrabase) and missed the fix entirely. All 5 save sites (Tidal / Deezer / Spotify Public / YouTube / Discovery Pool) now use the active primary source, matching what the automatic workers already do', page: 'sync' }, - ], - '2.37': [ - // --- April 21, 2026 (evening) --- - { date: 'April 21, 2026 (evening)' }, - { title: 'Fix Auto-Watchlist Ignoring Global Override Settings', desc: 'The scheduled auto-watchlist scan (not the manual one) called scan_watchlist_artists directly, which bypassed Global Override application. So if you disabled Albums or Live under Watchlist → Global Override, full albums and live tracks still got added to the wishlist during the nightly scan. Override logic now runs inside scan_watchlist_artists so every entry point respects it', page: 'watchlist' }, - { title: 'Fix Live Version Filter False Positives', desc: 'The \\blive\\b regex was too loose — it flagged any title with the word "live" regardless of context, so "What We Live For" by American Authors, "Live Forever" by Oasis, and similar verb uses got treated as live recordings. Tightened to require clear live-recording context: "(Live)", "- Live", "Live at/from/in/on/version/session/etc". Fixes both the watchlist/backfill track filter and the Library Maintenance Live/Commentary Cleaner', page: 'library' }, - ], - '2.36': [ - // --- April 21, 2026 --- - { date: 'April 21, 2026' }, - { title: 'Fix Metadata Cache Bar Duplicating on Findings Dashboard', desc: 'The "Metadata Cache · View Details" bar under the findings chips could stack into 2–6 copies if the dashboard refreshed while a cache-health fetch was still in flight. Each resolved fetch appended its own section. Now each fetch clears any existing bar before appending', page: 'library' }, - { title: 'Fix Discography Backfill Stalling When Repair Worker Paused', desc: 'Force-running a job via "Run Now" stalled forever when the master repair worker was paused. The job entered the scan function, logged its starting banner, then blocked on the first wait_if_paused check. Force-run now bypasses the master-pause — scheduled runs still respect it', page: 'library' }, - { title: 'Discography Backfill: 3-Option Fix Dialog', desc: 'Clicking Fix on a missing-track finding now prompts "Add to Wishlist", "Just Clear Finding", or "Cancel" instead of silently adding to wishlist. Bulk Fix shows the same prompt once for all selected backfill findings', page: 'library' }, - { title: 'Discography Backfill: Auto-Add to Wishlist Setting', desc: 'New opt-in setting in the Discography Backfill job config. When enabled, missing tracks are pushed straight to the wishlist during the scan AND a finding is created for the log. Default is off — you review and click Fix', page: 'library' }, - { title: 'Discography Backfill: Faster Batched Matching', desc: 'Each artist scan now pre-fetches the library albums + tracks once and matches in-memory — same fast path the Library and Artists pages use. Avoids thousands of per-track SQL queries on artists with big libraries', page: 'library' }, - { title: 'Discography Backfill: Rich Album Context per Finding', desc: 'Every finding now carries a full album dict (id, name, album_type, release_date, images, artists, total_tracks) matching the wishlist pipeline shape. No more generic "Add to Wishlist" loss of release metadata', page: 'library' }, - { title: 'Discography Backfill: Per-Artist Progress Logs', desc: 'Scan logs now show [N/50] Scanning ArtistName for each artist processed, with found-count or "no missing tracks" afterward. Makes it obvious whether the job is actually progressing' }, - - // --- April 20, 2026 (part 2) --- - { date: 'April 20, 2026 (evening)' }, - { title: 'Massively Faster Artist Detail Page Loads', desc: 'Artist discography completion checks used to fire hundreds of SQL queries per page load — 15+ fuzzy title/artist searches per album times 30 albums per artist. Now pre-fetches the artist\'s library albums and tracks ONCE upfront, then matches everything in-memory. Same matching logic and accuracy, roughly 100x fewer SQL round-trips. Applies to both the Library artist page and the Artists search page', page: 'library' }, - { title: 'Fix Reorganize All Ignoring Album Type', desc: 'Reorganize All was sending every album — EPs, singles, and compilations — into the "Albums" folder because the $albumtype template variable silently defaulted to "Album". The variable is now resolved from the album\'s record_type (with track-count fallback) so ${albumtype}s produces the expected Albums/Singles/EPs/Compilations split', page: 'library' }, - - // --- April 20, 2026 --- - { date: 'April 20, 2026' }, - { title: 'Discography Backfill Maintenance Job', desc: 'New library maintenance job that scans each artist in your library, fetches their full discography from metadata sources, and creates findings for any missing tracks. Review findings and click "Add to Wishlist" to queue them for download. Respects content filters (live/remix/acoustic/compilation) and release type filters. Opt-in, disabled by default', page: 'library' }, - { title: 'Multi-Artist Tagging Options', desc: 'Three new settings: configurable artist separator (comma/semicolon/slash), multi-value ARTISTS tag for Navidrome/Jellyfin multi-artist linking, and "Move featured artists to title" mode. All opt-in with defaults matching current behavior', page: 'settings' }, - { title: 'Reorganize All Albums for Artist', desc: 'New "Reorganize All" button in the enhanced library artist header. Processes all albums for an artist sequentially using the configured path template. Shows progress per album, continues on error', page: 'library' }, - { title: 'Enriched Downloads Page Cards', desc: 'Download cards now show album artwork thumbnail, artist name, album name, source badge, and quality badge — all pulled from existing metadata context. No extra API calls', page: 'downloads' }, - { title: 'Template Variable Delimiter Syntax', desc: 'Use ${var} syntax to append literal text to template variables: ${albumtype}s produces "Albums", "Singles", "EPs". Both $var and ${var} syntaxes work. Updated validation and hint text for all templates', page: 'settings' }, - { title: 'AcoustID Fix Action Prompt', desc: 'AcoustID mismatch findings now show a 3-option fix prompt (Retag/Re-download/Delete) instead of silently defaulting to retag. Works for both individual and bulk fix', page: 'library' }, - { title: 'Fix Sync Buttons on Undiscovered Playlists', desc: 'Sync buttons on ListenBrainz/Last.fm Radio playlists were visible before discovery due to the standalone mode handler resetting display:none on every WebSocket push. Now only restores buttons it specifically hid' }, - { title: 'Fix Wing It Tracks Added to Wishlist During Sync', desc: 'Wing It fallback tracks with no real metadata were being added to wishlist when they failed to match on the media server during playlist sync. Now skipped by checking the wing_it_ ID prefix' }, - { title: 'Fix iTunes Region-Restricted Albums', desc: 'iTunes API sometimes returns album metadata without song tracks for region-restricted releases. The empty result was cached permanently. Now tries fallback storefronts for actual songs, and skips caching empty results' }, - { title: 'Fix Disc Subfolder Missing on Single-Track Downloads', desc: 'Downloading a single track from search for a multi-disc album placed it without the Disc N/ subfolder. Now resolves total_discs from the album tracklist when not already known' }, - { title: 'Fix Allow Duplicate Tracks Setting Not Working', desc: 'The "Allow duplicate tracks across albums" setting was ignored during album download analysis. Tracks found in other albums were marked as owned and skipped. Now only checks ownership within the target album when duplicates are allowed' }, - { title: 'Stop slskd Log Spam When Not Active', desc: 'Download monitor and transfer cache were polling slskd every second during active downloads regardless of whether Soulseek was configured. Now skips slskd API calls entirely when Soulseek is not in the active download source' }, - { title: 'Fix AcoustID High-Confidence Skip', desc: 'AcoustID verification was letting wrong files through when the fingerprint score was high (0.95+) even with very low title/artist similarity. Now requires at least partial title or artist match before skipping verification' }, - { title: 'Fix Navidrome Multi-Library Import', desc: 'Full database refresh was importing albums from all Navidrome music folders even when only one was selected in settings. Now filters albums to the selected music folder using a cached album ID set' }, - { title: 'Fix Repair Worker Crash on Zero Interval', desc: 'Jobs with interval_hours set to 0 caused ZeroDivisionError in the repair worker staleness calculation. Now skips jobs with invalid intervals' }, - { title: 'Fix Playlist Mode Missing Metadata and Cover Art', desc: 'Playlist folder mode passed null album_info to metadata enhancement, causing the entire function to crash silently. All metadata was wiped from the file. Now normalizes null to empty dict and falls back to spotify_album context for cover art' }, - { title: 'Fix Unknown Artist Fixer Column Name', desc: 'The unknown_artist_fixer repair job crashed with "no such column: t.deezer_track_id". The tracks table uses deezer_id, not deezer_track_id' }, - { title: 'Fix Auto-Import Using Wrong Artist from Tags', desc: 'Auto-import trusted embedded file tags for artist names even when the parent folder clearly indicated the correct artist. Mixtapes tagged with DJ names (e.g. "Slim" instead of "2Pac") got organized under the wrong artist. Now uses parent folder structure as artist override when folder depth indicates an Artist/Album layout' }, - - // --- April 19, 2026 --- - { date: 'April 19, 2026' }, - { title: 'Fix Wishlist Albums Cycle Stuck at 1 Concurrent', desc: 'Auto-wishlist processing during the "albums" cycle was limited to 1 concurrent download even with higher configured settings. The max_concurrent=1 restriction is only needed for Soulseek folder-based album grabs, not individual wishlist track downloads. Albums cycle now uses the configured concurrency like singles' }, - { title: 'Fix Track Ownership False Positives Across Albums', desc: 'Track ownership check on the artist detail page now filters by album context. Previously "Thriller" from Thriller 25 would show as owned on every Michael Jackson album containing a track called Thriller. Now only matches within the specific album being checked' }, - { title: 'Fix Wing It Tracks Added to Wishlist via Button', desc: 'Wing It fallback tracks were skipped from wishlist on failed downloads but not when manually clicking "Add to Wishlist". Now consistently skipped across all paths' }, - { title: 'Fix Debug Info Showing Zero Counts', desc: 'Copy Debug Info button showed 0 for watchlist, wishlist, and automation counts due to calling get_db() instead of get_database(). Silent NameError was caught by try/except' }, - { title: 'Fix Album Track Lookup Hardcoded to Spotify', desc: 'Clicking an album on the Artists page to download tracks was hardcoded to use Spotify even when the user\'s primary metadata source was Deezer or iTunes. Now uses the configured primary source with Spotify as fallback' }, - { title: 'Fix Wishlist Splitting Albums by Track Artist', desc: 'Adding a multi-artist album (like a soundtrack) to wishlist was creating separate entries per track artist instead of keeping all tracks under the album artist. Now uses the album-level artist context when available to keep tracks grouped correctly' }, - { title: 'Fix Artist Search Case Sensitivity', desc: 'Artist search on the Artists page now normalizes all-lowercase queries to title case before hitting metadata APIs. Some APIs return fewer or no results for lowercase queries like "foreigner" vs "Foreigner"' }, - { title: 'Lidarr Download Source Now Production-Ready', desc: 'Lidarr is now a fully functional download source with complete orchestrator integration. Downloads appear in the UI, status polling works, cancellation works, and cleanup on shutdown works. Error messages are now visible in the download list. Removed "(Development)" label' }, - { title: 'Fix M3U Showing All Tracks as Missing', desc: 'M3U playlist files were generated before post-processing finished, so file paths pointed to download locations instead of final library paths. M3U is now regenerated from the backend after all post-processing completes, resolving real library paths from the DB' }, - { title: 'Fix AcoustID Retag Not Writing to File', desc: 'The AcoustID mismatch "Retag" fix action was only updating the database record without writing corrected tags to the actual audio file. Now writes title and artist tags to the file using Mutagen after updating the DB' }, - { title: 'Fix Downloads Badge Dropping to 300', desc: 'Downloads nav badge showed the correct count from WebSocket but dropped to max 300 after opening the Downloads page because it recounted from a truncated local array. Badge now stays accurate from the server-side count' }, - { title: 'Fix Server Playlist Find & Add Position', desc: 'When using "Find & add" on server playlists with Plex, the track was always appended to the end instead of inserted at the correct position. Now moves the track to the right slot after adding' }, - { title: 'Smarter Fix Modal Search Results', desc: 'The discovery Fix modal now sorts search results to prioritize standard album versions over live recordings, remixes, covers, soundtracks, remasters, and deluxe editions. Previously the first result was often a live or remix version instead of the original studio track' }, - { title: 'Unmatch Discovery Tracks', desc: 'Found tracks in playlist discovery now have a red ✕ button to remove the match. Sets the track back to Not Found so it won\'t be downloaded. For mirrored playlists, the unmatch persists in the DB and is respected on re-discovery runs' }, - { title: 'Customizable Music Video Naming', desc: 'Music video file naming is now configurable via a path template in Settings → Library → Paths & Organization. Default unchanged (Artist/Title-video.mp4). Remove "-video" from the template to get clean filenames. Available variables: $artist, $artistletter, $title, $year', page: 'settings' }, - { title: 'Fix Soulseek Log Spam', desc: 'The "Clean Search History" automation no longer tries to connect to slskd when Soulseek is not the active download source, eliminating noisy connection error logs for users who don\'t use Soulseek' }, - { title: 'Auto Wing It Discovery Fallback', desc: 'When playlist discovery fails to match a track on any metadata API (Spotify, Deezer, iTunes, etc.), the track now automatically falls back to Wing It mode instead of being marked "Not Found". Stub metadata is built from the raw source title and artist, and the track flows through the normal download pipeline via Soulseek. Amber "Wing It" badge distinguishes these from API-matched tracks. Works across all discovery sources: YouTube, Tidal, Deezer, Beatport, ListenBrainz, and mirrored playlists. Wing It stubs persist in the DB for mirrored playlists and are re-attempted on future discovery runs so real matches can replace them' }, - { title: 'Fix Library Page Crash on All Filter', desc: 'Library page could crash with "No artists found" when viewing all artists if any artist had a non-string soul_id. Individual letter filters worked because the problematic artist wasn\'t in those results. Card rendering is now fault-tolerant — one bad artist card can\'t take down the whole page', page: 'library' }, - { title: 'Fix CI Test Failures', desc: 'Fixed test suite failures caused by incomplete dummy config managers missing get_active_media_server() and script.js read encoding on non-UTF-8 locales' }, - - // --- April 18, 2026 --- - { date: 'April 18, 2026' }, - { title: 'Live Log Viewer', desc: 'New Logs tab on the Settings page — real-time terminal-style log viewer with color-coded log levels. Filter by DEBUG/INFO/WARNING/ERROR, search logs in real-time, switch between log files (app, post-processing, acoustid, source reuse). Auto-scroll, copy, clear. Live WebSocket updates every 0.5s. Smart level detection works on both logger output and print statements', page: 'settings' }, - { title: 'ReplayGain Post-Processing', desc: 'Optional ReplayGain tag analysis during post-processing. Enable in Settings → Library → Post-Processing. Analyzes loudness via ffmpeg and writes track-level gain/peak tags. Runs before lossy copy so both files get tagged. Off by default' }, - { title: 'Fix Your Albums Using Playlist Modal', desc: 'Albums in the Discover page "Your Albums" section now open with the proper album-style download modal instead of the playlist-style modal. Shows artist image, album art, and uses album download context for correct file organization', page: 'discover' }, - { title: 'Fix Tool Help Modal Not Closable', desc: 'The help "?" modal on automation triggers/actions could not be closed if the Tools page hadn\'t been visited first. Close button, backdrop click, and Escape key now work from any page' }, - { title: 'Fix Spotify OAuth Port Steal in Docker', desc: 'On fresh installs, Spotify auth probe silently started an HTTP server that stole port 8008 (crash loop) or bound loopback-only on 8888 (unreachable from host). Now skips the probe when no cached token exists' }, - { title: 'Genre Whitelist', desc: 'Filter junk genre tags (artist names, radio shows, playlist names) from enrichment. Enable strict mode in Settings → Library Preferences → Genre Whitelist. 272 curated default genres, fully customizable — add, remove, search, reset. Applied across all 10 enrichment sources. Off by default', page: 'settings' }, - { title: 'Per-Artist Watchlist Scan Source', desc: 'Override which metadata provider (Spotify, Deezer, Apple Music, Discogs) is used when scanning a specific watchlist artist for new releases. Source selector in the artist config modal only shows providers the artist has enrichment matches for. Global default unchanged unless explicitly overridden', page: 'watchlist' }, - { title: 'Standalone Full Refresh', desc: 'Full Refresh now works for SoulSync Standalone mode — clears all soulsync library records and rebuilds from audio file tags in the output folder. Previously did nothing for standalone users', page: 'tools' }, - { title: 'Folder Terminology Rebrand', desc: 'Download Path → Input Folder, Transfer Path → Output Folder, Staging Path → Import Folder. All UI labels, docs, help text, and error messages updated for clarity. No functional changes — actual paths and config keys unchanged' }, - { title: 'Enhanced Copy Debug Info', desc: 'Copy Debug Info button now includes ffmpeg version, runner type, Discogs status, wishlist count, music library paths, music videos dir, hybrid source priority, lossy copy config, auto import status, and a log file listing with sizes. Import path bug fixed. Library counts now match dashboard. Footer links to GitHub Issues', page: 'help' }, - { title: 'Troubleshooting Docs Section', desc: 'New Help page section with log file reference table, log level guide, Copy Debug Info walkthrough, common issues FAQ, and issue reporting checklist', page: 'help' }, - { title: 'Log Level Moved to Advanced Tab', desc: 'Log Level dropdown moved from Downloads tab to Settings → Advanced → Logging for better organization' }, - { title: 'Fix AcoustID Scanner Fix Action', desc: 'AcoustID mismatch "Fix" button was failing with a uuid error. Caused by a redundant local import shadowing the module-level import in Python\'s scoping rules' }, - { title: 'Fix Duplicate Detector Ignoring Allow Duplicates', desc: 'The Duplicate Detector repair job now respects the global "Allow duplicate tracks across albums" setting. Previously flagged cross-album duplicates regardless of the toggle' }, - { title: 'Fix Single Track Search Downloads Using Album Template', desc: 'Clicking a single track in search results and downloading it now uses the singles path template instead of the album template. The modal correctly showed SINGLE but the backend treated it as an album download' }, - { title: 'Fix Liked Songs Showing as YouTube', desc: 'Spotify Liked Songs playlist was misidentified as YouTube in the download modal hero section due to missing spotify: prefix detection' }, - { title: 'Fix Metadata Crash on Playlist Downloads', desc: 'Playlist and single track downloads could crash metadata enhancement with "NoneType has no attribute get" when album_info was None' }, - { title: 'Fix Library Scan Button Stuck on Stop', desc: 'Dashboard library scan polling checked for "completed" but backend sets "finished". Button now resets correctly and stats refresh on completion' }, - { title: 'Fix Deep Scan Reporting Stale Records as Failed', desc: 'Stale record removals during deep scan were counted as "failed" instead of "successful" in the completion message' }, - { title: 'Fix Settings Page Tab Flash', desc: 'Settings page no longer briefly shows all tabs on first load — tab filtering now runs before async data loading' }, - { title: 'Improved Deep Scan Logging', desc: 'Per-artist log lines now show "0 new tracks (150 existing updated)" instead of misleading "0 tracks". Completion message shows "library up to date" when nothing is new' }, - { title: 'Faster Standalone Verify', desc: 'Standalone verify button now stops counting after 10 audio files instead of 100, reducing verification time from 60+ seconds to near-instant on large libraries' }, - { title: 'MusicBrainz Search Tab', desc: 'New search tab in Enhanced and Global search — find tracks and albums on MusicBrainz\'s community database. Cover art from Cover Art Archive. Click results to open download modal with full tracklist. Finds obscure tracks that Spotify/Deezer/iTunes miss', page: 'downloads' }, - { title: 'Fix Library Page Crash on All Filter', desc: 'Library page could crash with "No artists found" when viewing all artists if any artist had a non-string soul_id. Individual letter filters worked because the problematic artist wasn\'t in those results. Card rendering is now fault-tolerant — one bad artist card can\'t take down the whole page', page: 'library' }, - - // --- April 17, 2026 --- - { date: 'April 17, 2026' }, - { title: 'SoulSync Standalone Library', desc: 'New "Standalone" server option — manage your library without Plex, Jellyfin, or Navidrome. Downloads and imports write directly to the library database with pre-populated enrichment IDs. Deep scan finds untracked files and cleans stale records. Select in Settings → Connections', page: 'settings' }, - { title: 'Auto-Import', desc: 'Background import folder watcher that automatically identifies and imports music. Three strategies: audio tags, folder name parsing, and AcoustID fingerprinting. Confidence-gated: 90%+ auto-imports, 70-90% queued for review, below 70% left for manual. Enable on the Import page Auto tab', page: 'import' }, - { title: 'Wishlist Nebula', desc: 'Wishlist redesigned as an interactive artist orb visualization. Each artist is a glowing orb with their photo — album fans and single moons orbit around them. Click orbs to expand, download albums/singles directly. Processing state shows live progress', page: 'wishlist' }, - { title: 'Automation Group Management', desc: 'Rename, delete, and bulk-toggle automation groups. Drag-and-drop automations between groups. Right-click group headers for context menu', page: 'automations' }, - { title: 'Bidirectional Artist Sync', desc: 'Artist Sync button now pulls new content from your media server AND removes stale library entries no longer on the server. Deep scan mode fetches full metadata for new tracks', page: 'library' }, - { title: 'Server Playlists — Synced vs Unsynced', desc: 'Server playlist view now shows all playlists from your media server with clear visual separation between synced and unsynced playlists', page: 'sync' }, - { title: 'Provider-Agnostic Discovery', desc: 'Similar artist matching, discovery pool, and incremental updates now work with any configured metadata source (Spotify, iTunes, Deezer) instead of requiring Spotify. Falls back through sources in priority order', page: 'watchlist' }, - { title: 'Live Sidebar Badges', desc: 'Watchlist and Wishlist sidebar nav items show live count badges that update from WebSocket pushes' }, - { title: 'Fix Source ID Embedding', desc: 'Critical fix — all source ID tags (Spotify, MusicBrainz, Deezer, AudioDB) were silently skipped on every download due to a missing function parameter. Tags now embed correctly again' }, - { title: 'Fix Watchlist Scan False Failures', desc: 'Artists with no new releases in the lookback window were incorrectly reported as scan failures. Empty discography now correctly treated as success' }, - { title: 'Fix Wishlist Album Remove', desc: 'Removing albums from the Wishlist Nebula now works — API accepts album_name as fallback when album_id is unavailable' }, - { title: 'Fix Soulseek Timeout Spam', desc: 'Dashboard stats and download status endpoints no longer poll slskd when Soulseek is not the active download source or is known to be disconnected. Eliminates connection timeout errors every 10 seconds for users who have a slskd URL configured but use YouTube/Tidal/etc.' }, - { title: 'Fix Soulseek Search Missing Album Name', desc: 'Soulseek search queries now include the album name (Artist + Album + Track) as the first search attempt for all download sources. Previously this was excluded for Soulseek-only mode, causing wrong-artist downloads when an artist name matched an album folder in another user\'s library' }, - { title: 'Reject Junk Artist Soulseek Results', desc: 'Soulseek search results from "Various Artists", "VA", "Unknown Artist", and "Unknown Album" folders are now automatically rejected. These compilation/junk folders almost never contain properly tagged files for the target artist' }, - { title: 'Clear Wishlist Cancels Downloads', desc: 'Clearing the wishlist now also cancels any active wishlist download batch. Previously the download queue would keep running after the wishlist was cleared' }, - { title: 'Downloads Batch Panel', desc: 'Downloads page now shows a batch context panel on the right side. Each active batch (wishlist, sync, album download) gets a color-coded card with progress, cancel button, and expandable track list. Color indicators on download rows link them to their batch. Completed batch history shows the last 7 days', page: 'active-downloads' }, - { title: 'Fix Unknown Artist on Wishlist Downloads', desc: 'Adding tracks to wishlist from a playlist download modal was storing "Unknown Artist" as the artist context. Now resolves the artist per-track from the track\'s own metadata instead of the playlist-level artist which is only set for album downloads' }, - { title: 'Fix Download Modal Freezing Mid-Download', desc: 'Download modals (wishlist, sync, album) would freeze and stop updating after the first track completed. Caused by M3U auto-save firing every 2 seconds during downloads, exhausting Flask server threads. Now saves M3U once on completion only' }, - { title: 'Auto-Import Improvements', desc: 'Recursive import folder scan (any folder depth), single file support, expandable track match details, stats bar with filters, Scan Now button, Approve All / Clear History batch actions. Tag-based identification preferred over weak metadata matches. AcoustID fallback for untagged files. Race condition fix prevents duplicate processing', page: 'import' }, - { title: 'Album Delete with File Removal', desc: 'Enhanced library album delete now offers "Delete Files Too" option alongside "Remove from Library" — deletes audio files from disk and cleans up empty album folders', page: 'library' }, - - // --- April 15, 2026 --- - { date: 'April 15, 2026' }, - { title: 'Dashboard Library Status Card', desc: 'Smart card on the Dashboard showing your library state — server connection, track counts, last refresh time. Guides new users through setup, shows empty-library prompts, and lets you trigger a scan directly from the dashboard', page: 'dashboard' }, - { title: 'AcoustID Scanner Upgrade', desc: 'Now scans your full library (not just Transfer) to detect wrong downloads. Actionable fixes: retag with correct metadata, re-download the right track, or delete the wrong file. Enabled by default, runs daily' }, - { title: 'Tools Page', desc: 'All tool cards (Database Updater, Quality Scanner, Duplicate Cleaner, Retag, Backups, Cache, etc.) and Library Maintenance moved from the Dashboard to a dedicated Tools page in the sidebar. Dashboard shows a quick-link card', page: 'tools' }, - { title: 'Watchlist & Wishlist Sidebar Pages', desc: 'Watchlist and Wishlist promoted from modals to full sidebar pages. All features preserved — artist grid, scan controls, batch operations, live activity, countdown timers. Header buttons now navigate to the pages', page: 'watchlist' }, - { title: 'Picard-Style MusicBrainz Album Consistency', desc: 'Recording MBIDs now pulled from the matched release tracklist instead of independent searches. Batch-level artist name used for stable cache keys. Post-batch consistency pass rewrites album-level tags on all files to guarantee identical MusicBrainz IDs — prevents Navidrome album splits' }, - { title: 'Fix Spotify API Leaking When Deezer/iTunes is Primary', desc: 'Spotify was being called for watchlist album scanning, similar artist discovery, repair jobs, and the Artists page search even when another source was set as primary. All data-fetching now respects the configured primary source. Spotify playlist sync is unaffected' }, - { title: 'Fix OAuth Callback Port Hardcoding', desc: 'Custom callback ports (SOULSYNC_SPOTIFY_CALLBACK_PORT / SOULSYNC_TIDAL_CALLBACK_PORT) are now respected in auth instruction pages and log messages instead of always showing 8888. Added startup diagnostics logging for callback port binding' }, - { title: 'Fix Allow Duplicates Setting Not Saving', desc: 'The "Allow duplicate tracks across albums" toggle was never persisted — it silently reset to ON on every page reload. Now saves correctly' }, - { title: 'Fix Wishlist Dropping Cross-Album Tracks', desc: 'Wishlist cleanup was removing same-titled tracks from different albums even when Allow Duplicates was enabled. Cleanup now respects the setting — same song from different albums can coexist in the wishlist' }, - { title: 'Fix "Replace Lower Quality" Setting Not Persisting', desc: 'The import section appeared twice in the settings save payload — the second instance (with only staging_path) overwrote the first (with replace_lower_quality). Merged into a single block' }, - { title: 'Inbound Music Request API', desc: 'New POST /api/v1/request endpoint — trigger downloads from Discord bots, Home Assistant, curl, or any external tool. Async with status polling and optional notify_url callback. New "Webhook Received" automation trigger and "Search & Download" action in the Automation Hub' }, - { title: 'Fix Spotify Enrichment Worker Infinite Loop', desc: 'Artists with an existing Spotify ID but no match status got stuck in the enrichment queue — the worker processed them every 3 seconds forever without marking them as done. Now correctly marks them as matched' }, - { title: 'Reject Qobuz 30-Second Samples', desc: 'Qobuz previews (30s samples for tracks requiring a subscription or region-restricted) are now detected and rejected. Checks the API sample flag before downloading, and validates file duration after download as a safety net' }, - - // --- April 14, 2026 --- - { date: 'April 14, 2026' }, - { title: 'Fix Import Files Ignoring Path Template', desc: 'Files matched from the import folder were copied to the output root with their original filename instead of applying the configured path template. Post-processing now receives full artist/album context for import matches' }, - - // --- April 4, 2026 --- - { date: 'April 4, 2026' }, - { title: 'Artist Map — Visualize Your Music Universe', desc: 'Three interactive canvas modes: Watchlist Constellation (your artists + similar), Genre Map (browse by genre with sidebar), and Artist Explorer (deep-dive any artist). Offscreen buffer rendering handles 1000+ nodes', page: 'discover' }, - { title: 'Artist Explorer — On-the-Fly Discovery', desc: 'Explore any artist even if not in your library — fetches similar artists from MusicMap in real-time, stores results for instant future visits. Invalid names validated against Spotify/iTunes', page: 'discover' }, - { title: 'Genre Map — Full Artist Counts', desc: 'Genre map now shows all artists per genre (no caps). Ring packing layout handles large genres instantly. Genre sidebar for quick switching', page: 'discover' }, - { title: 'Artist Map Caching', desc: 'Server-side 5-minute cache on all artist map endpoints — switching genres and reopening maps is instant. Auto-invalidates on watchlist changes and scans' }, - { title: 'Image Proxy for Canvas Rendering', desc: 'Server-side image proxy solves CORS issues for canvas — Deezer, Last.fm, and Discogs images now render on Artist Map bubbles' }, - - // --- April 3, 2026 --- - { date: 'April 3, 2026' }, - { title: 'Your Artists on Discover', desc: 'Aggregates liked/followed artists from Spotify, Tidal, Last.fm, and Deezer. Auto-matched to all metadata sources. Click for artist info modal with bio, genres, stats, and watchlist toggle', page: 'discover' }, - { title: 'Deezer OAuth', desc: 'Full Deezer OAuth integration for user favorites and playlists. Configure in Settings → Connections' }, - { title: 'Failed MB Lookups Manager', desc: 'Browse, search, and manually match failed MusicBrainz lookups from the Cache Health modal. Search MusicBrainz directly and save matches' }, - { title: 'Explorer Controls Redesign', desc: 'Playlist Explorer controls redesigned with prominent Explore button, icons, status badges, auto-refresh, and discover from Explorer', page: 'playlist-explorer' }, - { title: '$discnum Template Variable', desc: 'Unpadded disc number for multi-disc album path templates — e.g. Disc 1, Disc 2' }, - { title: 'Fix Album Folder Splitting', desc: 'Collab albums no longer scatter tracks across multiple folders — $albumartist uses album-level artist consistently' }, - { title: 'Fix Watchlist Rate Limiting', desc: 'Watchlist scans fetch only newest albums (~90% fewer API calls). Configurable API interval. Better Retry-After extraction' }, - { title: 'Fix Media Player Collapsing', desc: 'Media player no longer collapses in the sidebar on short viewports and mobile devices' }, - - // --- April 2, 2026 --- - { date: 'April 2, 2026' }, - { title: 'Discogs Integration', desc: 'New metadata source — enrichment worker, fallback source, enhanced search tab, watchlist support, cache browser. 400+ genre/style taxonomy', page: 'dashboard' }, - { title: 'Webhook THEN Action', desc: 'Send HTTP POST to any URL when automations complete — Gotify, Home Assistant, Slack, n8n', page: 'automations' }, - { title: 'API Rate Monitor', desc: 'Real-time speedometer gauges for all enrichment services on Dashboard. Click any gauge for 24h history', page: 'dashboard' }, - { title: 'Configurable Concurrent Downloads', desc: 'Set max simultaneous downloads (1-10) in Settings. Soulseek albums stay at 1 for source reuse' }, - { title: 'Streaming Search Sources', desc: 'Apple Music results stream progressively instead of blocking for 9+ seconds' }, - { title: 'Track Provenance Through Transcoding', desc: 'Download source info preserved when Blasphemy Mode converts FLAC to lossy (#245)' }, - - // --- April 1, 2026 --- - { date: 'April 1, 2026' }, - { title: 'Wing It Mode', desc: 'Download or sync playlists without metadata discovery — uses raw track names directly' }, - { title: 'Global Search Bar', desc: 'Spotlight-style search from any page — press / or Ctrl+K. Full enhanced search with source tabs', page: 'downloads' }, - { title: 'Redesigned Notifications', desc: 'Compact pill toasts, notification bell with unread badge, history panel with last 50 notifications' }, - { title: 'Track Redownload & Source Info', desc: 'Fix mismatched downloads from the enhanced library view. Source Info shows download provenance with blacklist option', page: 'library' }, - { title: 'Block Artists from Discovery', desc: 'Permanently exclude artists from all discovery playlists — hover any track and click ✕', page: 'discover' }, - { title: 'MusicBrainz Cache in Browser', desc: 'MusicBrainz cache now visible in Cache Browser with clear and clear-failed-only options' }, - - // --- Earlier in v2.2 --- - { date: 'March 2026' }, - { title: 'Server Playlist Manager', desc: 'Compare source playlists against your media server — find missing tracks, swap wrong matches, remove extras', page: 'sync' }, - { title: 'Sync History Dashboard', desc: 'Recent syncs as cards on Dashboard — click for per-track match details with confidence scores' }, - { title: 'Playlist Explorer', desc: 'Expand playlists into visual discovery trees of albums and discographies', page: 'playlist-explorer' }, - { title: 'Enhanced Library Manager', desc: 'Inline tag editing, bulk operations, write-to-file, and per-artist library sync', page: 'library' }, - { title: 'Automation Signals', desc: 'Chain automations together using fire/receive signals with cycle detection', page: 'automations' }, - { title: 'Multi-Source Search Tabs', desc: 'Compare results from Spotify, iTunes, and Deezer side by side', page: 'downloads' }, - { title: 'Rich Artist Profiles', desc: 'Full-bleed hero section with bio, stats, genres, and service links', page: 'artists' }, - { title: 'Spotify API Rate Limit Improvements', desc: 'Cached discography lookups, eliminated duplicate calls, enrichment workers auto-pause during downloads' }, + '2.5.5': [ + { date: 'May 17, 2026 — 2.5.5 release' }, + { title: 'Manual Library Match', desc: 'stop SoulSync from re-downloading tracks it already has. new centralized tool (Tools page → Manual Library Match, or Sync page → Library Match button) lets you search your wishlist / sync history on the left and your library on the right, then link them. once matched, that source track is permanently skipped in wishlist cleanup and the download analysis loop — even when force download is on. manage all your matches in one place with remove support.', page: 'tools' }, ], }; @@ -3827,6 +3498,161 @@ const WHATS_NEW = { // Section shape: { title, description, features: [bullet strings], // usage_note?: 'optional hint shown at the bottom' } const VERSION_MODAL_SECTIONS = [ + { + title: "Qobuz Playlist Sync", + description: "Qobuz joins Tidal and Deezer as a first-class playlist sync source on the Sync page. Browse your Qobuz playlists and Favorite Tracks, run them through the same discovery flow as Tidal, sync the resulting Spotify-matched tracks, and queue downloads — same multi-step pipeline you already know.", + features: [ + "new Qobuz tab on the Sync page, listed between Deezer and Deezer Link", + "lists your Qobuz user playlists plus a Favorite Tracks entry (same virtual-playlist treatment Tidal gets)", + "click any card to fire discovery (Spotify-preferred, your primary metadata fallback otherwise), then sync or download just like Tidal / Deezer playlists", + "uses the Qobuz auth token you already configured for downloads — no extra connection step", + ], + usage_note: "Sync → Qobuz → 🔄 Refresh", + }, + { + title: "Earlier in v2.5", + description: "highlights from the 2.5.x cycle that landed just before 2.6.0: new release-based download sources, HiFi instance probing, Jellyfin full-refresh repairs, transient SQLite retries, and tighter Album Completeness artist matching.", + features: [ + "Torrent and Usenet are now available as Prowlarr-backed download sources with release staging, live progress, and source-aware history labels", + "HiFi instances are probed by actually checking whether they can return a playable manifest, including legacy public hifi-api instances", + "Jellyfin full refresh repairs older media databases before importing tracks, so stale schemas no longer drop every track row", + "Cache maintenance and full refresh retry transient SQLite disk I/O errors instead of failing the whole job on the first attempt", + "Album Completeness rejects same-title releases from the wrong artist before importing anything", + "Import album search now labels each card with the source that served it and warns when results came from a fallback instead of your primary", + ], + usage_note: "browse the What's New panel for the full 2.5.x changelog", + }, + { + title: "Torrent and Usenet Are Now Live Download Sources", + description: "the long-awaited payoff. Two new entries in the Download Source dropdown — Torrent Only and Usenet Only — both backed by your Prowlarr indexers. Searches go through Prowlarr filtered by protocol, picked releases get shipped to your configured torrent client (qBit / Transmission / Deluge) or usenet client (SABnzbd / NZBGet), and the resulting files flow through SoulSync's matching pipeline like any other source.", + features: [ + "• new 'Torrent Only (via Prowlarr)' and 'Usenet Only (via Prowlarr)' options in Settings → Downloads → Download Source", + "• both sources also available in hybrid mode — drop them into the priority list alongside soulseek / tidal / etc.", + "• shared archive_pipeline walks the downloaded directory and extracts any .zip / .rar / .tar archives the downloader didn't already unpack (with path-traversal protection)", + "• torrent state polling handles the qBit / Transmission / Deluge state quirks uniformly; usenet polling covers the verify / repair / unpack phases", + "• picking a torrent or usenet source on the Downloads tab now shows a redirect card pointing to the Indexers & Downloaders tab where the actual config lives", + "• caveat: SoulSync needs filesystem access to the torrent / usenet client's save_path. fine for everything-on-one-box; remote downloader hosts need a future sync step", + ], + usage_note: "Settings → Downloads → Download Source → Torrent / Usenet Only", + }, + { + title: "Usenet Client Adapters (SABnzbd, NZBGet)", + description: "third phase of the torrent + usenet rollout. SoulSync now also talks to the two big usenet downloaders through a sibling adapter contract. Prowlarr + torrent + usenet are all stood up — next commit wires them together into actual download sources.", + features: [ + "• supports SABnzbd (API-key auth) and NZBGet (JSON-RPC basic auth)", + "• new Usenet Client section on the Indexers & Downloaders tab; client picker swaps the credential fields automatically (API key vs username + password)", + "• state mapping covers the verify / repair / unpack phases unique to usenet", + "• category override so SoulSync's NZBs land in a predictable post-processing folder", + "• Test Connection probes the live API", + "• next commit wires Prowlarr → adapter → archive extraction → match so the new sources fully download", + ], + usage_note: "Settings → Indexers & Downloaders → Usenet Client", + }, + { + title: "Torrent Client Adapters (qBit, Transmission, Deluge)", + description: "second phase of the torrent + usenet rollout. SoulSync now speaks the three big torrent client APIs through one uniform adapter — pick which client you use and SoulSync handles the auth and protocol quirks for you.", + features: [ + "• supports qBittorrent (WebUI v2), Transmission (RPC), Deluge 2.x (JSON-RPC)", + "• new Torrent Client section on the Indexers & Downloaders tab", + "• single client type picker — switching between clients is a dropdown change, no code path divergence", + "• per-client credential fields with hints (qBit / Transmission use username + password, Deluge uses password only)", + "• optional category/label and save-path overrides so SoulSync's torrents are easy to spot in your client", + "• Test Connection probes the live WebUI and confirms auth works", + "• no downloads triggered yet — the wire-up to Prowlarr search results lands once usenet client adapters ship", + ], + usage_note: "Settings → Indexers & Downloaders → Torrent Client", + }, + { + title: "Prowlarr Integration (Phase 1 of Torrent + Usenet)", + description: "first commit toward torrent and usenet download sources. SoulSync can now talk to your Prowlarr instance and pull the list of configured indexers, setting up the search surface that the torrent and usenet clients will plug into next.", + features: [ + "• new Indexers & Downloaders tab on the Settings page", + "• point SoulSync at Prowlarr with a URL and API key — same kind of setup as Lidarr", + "• Test Connection button confirms Prowlarr is reachable and authenticated", + "• Refresh Indexer List pulls the full list of indexers Prowlarr is currently managing (torrent + usenet, enabled state, privacy level)", + "• optional indexer-ID allowlist if you want SoulSync to only search a subset", + "• no downloads yet — torrent and usenet client adapters land in the next commits", + ], + usage_note: "Settings → Indexers & Downloaders → Prowlarr", + }, + { + title: "MusicBrainz Is Now a First-Class Metadata Source", + description: "MusicBrainz was already available as an optional search tab, but it wasn't selectable as your primary metadata source. now it is — switch to it in Settings → Metadata Source and the whole app routes through it.", + features: [ + "• always available — no account, no API key, no token needed", + "• rate-limited to 1 req/sec at the client layer, consistent with MusicBrainz's terms", + "• covers the full primary-source interface: search, album/track/artist lookup, top tracks, artist albums, discography", + "• watchlist scanner now backfills MusicBrainz artist IDs alongside Spotify / iTunes / Deezer in the similar artists table", + "• discover hero, artist map, and personalized playlists all source from MusicBrainz IDs when it's the active primary", + "• cover art served via Cover Art Archive — no extra API calls, browser fetches the URL directly", + "• fallback source logic in Settings and the registry now reads from source priority order instead of hardcoding 'deezer'", + ], + usage_note: "Settings → Metadata → Primary Source → MusicBrainz", + }, + { + title: "Live Recordings Stop False-Quarantining", + description: "github issue #607: live recordings were quarantining as 'Version mismatch: expected ... (live) but file is ... (original)' because MusicBrainz often stores live recordings with bare titles — venue annotations live on the release entity, not the recording entity itself. AcoustID's fingerprint correctly identified the live recording, but the title-text comparison flagged it as wrong.", + features: [ + "• new escape valve in the version-mismatch gate: when one side is 'live' and the other is bare 'original' + fingerprint score >= 0.85 + bare titles match + artist matches, accept", + "• other version mismatches (instrumental vs vocal, remix vs original, acoustic vs studio, demo) stay strict — those have distinct fingerprints AND MB always annotates them in the recording title", + "• fingerprint-score floor of 0.85 is stricter than the existing 0.80 minimum, so the escape valve only fires when AcoustID is more confident than its own threshold", + "• logic lifted to pure helper core/matching/version_mismatch.py with 23 boundary tests + the existing instrumental/remix tests still pin those cases as strict mismatches", + "• audio-mismatch failure message now reports the actual best-matching recording's strings, not recordings[0]'s — prior code mixed strings from one candidate with scores from another, producing nonsense reasons like \"identified as '' by '' (artist=100%)\"", + ], + usage_note: "no settings to change — applies on next download attempt of a live recording", + }, + { + title: "Quarantine Modal: Apostrophe Bug Fixed + Source Info Added", + description: "github issue #608: quarantined files with an apostrophe in the filename couldn't be Approved or Deleted — the unescaped quote broke the inline JS in the onclick handler, so the click silently no-op'd. Plus a UX add: each entry now shows the source uploader and original soulseek filename when available.", + features: [ + "• id is now wrapped via escapeHtml(JSON.stringify(id)) so apostrophes (and backslashes, double quotes, unicode, newlines) round-trip safely through the HTML attribute → JS string boundary", + "• quarantine entry expanded view shows source uploader (username) and original soulseek filename when the sidecar carries that context — helps trace which uploader the bad file came from", + "• degrades gracefully when the sidecar lacks the source fields (legacy thin sidecars) — fields just hide", + ], + usage_note: "open Library History → Quarantine tab", + }, + { + title: "Reorganize Can Now Use Your Embedded File Tags Instead Of An API Lookup", + description: "github issue #592: for users with a well-enriched, well-tagged library, doing a fresh metadata-source api lookup at reorganize time can drift from what's already on the file — provider naming inconsistencies, missing album-level metadata for niche releases. new optional mode reads each file's embedded tags as the source of truth instead. zero api calls.", + features: [ + "• new \"metadata mode\" dropdown on the per-album reorganize modal + bulk reorganize-all modal: 'API metadata' (default, current behavior) vs 'Embedded file tags'", + "• tag-mode reads via the same mutagen path the audit-trail modal uses — id3 / vorbis / mp4 all covered", + "• handles id3-style \"5/12\" track + disc shapes, multi-value Artists tags, year normalization across 5 date formats, releasetype canonical tokens (album / single / ep / compilation)", + "• partial-album reorganize respects the \"totaldiscs\" tag — disc 1 of a 2-disc set still routes into Disc 1/ subfolder", + "• each track's own album metadata flows through post-process (not a shared one), so per-file enrichment differences are preserved", + "• default stays 'API metadata' so existing reorganize pipelines are completely untouched — opt-in only", + "• per-track unmatched reasons surface in the preview when a file's tags are missing essentials, so you can see exactly which tracks need attention", + ], + usage_note: "artist detail → album → reorganize → 'metadata mode' dropdown; or pass `mode: 'tags'` to the api endpoint", + }, + { + title: "Dashboard Cursor-Following Accent Blob", + description: "the bento dashboard now has a subtle two-layer accent glow that follows your cursor as you sweep across cards. card backgrounds are darker too for better contrast.", + features: [ + "• soft halo (large + blurred) lerps toward your cursor with a delay — liquid trailing feel", + "• brighter inner core (smaller + screen-blended) gives the blob a punchy center", + "• both layers gently pulse on different rhythms (5.5s halo, 3.7s core) so it feels alive", + "• mouse leaves the cards or sits in a gap → blob freezes for 1.5s then drifts back to grid center", + "• container backgrounds darkened to near-black with stronger borders for contrast", + "• fully disabled by the existing reduce visual effects setting", + "• performant: only animates while moving, batches getBoundingClientRect reads per frame", + ], + usage_note: "nothing to configure — visit the home page; toggle reduce visual effects in settings to disable", + }, + { + title: "Dashboard Got A Bento Redesign", + description: "old dashboard had a lot of wasted space and sections fighting for attention. rebuilt as a bento grid with accent-tinted cards and subtle motion.", + features: [ + "• every section lives in its own card with a soft accent-tinted glow matching your theme color", + "• cards fade up on first paint with a staggered reveal — feels alive without being noisy", + "• layout adapts: 3-col bento on desktop (≥1500px), 2-col on laptop, 2-col tighter on tablet, single-column on mobile", + "• enrichment service gauges ride a single 10-tile row at desktop and wrap to 5 / 4 / 3 / 2 as space tightens", + "• system stats render 3-up over 2 rows so all 6 metrics fit without scrolling", + "• recent syncs stack vertically inside their card so you can scan them at a glance", + "• every existing button + status indicator + id preserved — same dashboard, just laid out properly", + ], + usage_note: "nothing to configure — visit the home page to see it", + }, { title: "Big Sync Sessions No Longer Wedge After 2-3 Hours", description: "github issue #499 (bafoed): downloading a big initial sync from spotify playlists worked for 2-3 hours then silently stopped. 3 active tasks stuck in \"searching\" state, replaced every ~10 min, slskd ui showed no actual activity. only fix was restarting the container.", @@ -4032,278 +3858,19 @@ const VERSION_MODAL_SECTIONS = [ ], }, { - title: "Earlier in v2.4 — Reorganize, Search, Sync polish", - description: "highlights from the 2.4.0 cycle that landed before this patch.", + title: "Earlier in v2.4", + description: "highlights from the 2.4 cycle.", features: [ - "• reorganize is now a queue with a live status panel — spam-click all you want, items run one at a time and you can keep browsing", - "• search page got a row of source icons above the bar — typing only searches the active source instead of fanning out to all of them", - "• per-query source cache + cache dots — switching back to a source you already searched is instant", - "• fix: \"maduk — leave a light on\" on tidal was downloading tom walker\'s song of the same name with maduk\'s metadata embedded — tightened the candidate artist gate and acoustid verification", - "• tidal: rejects silent quality downgrades (320kbps when you asked for hires)", - "• spotify: bumped post-ban cooldown from 5 to 30 minutes — first call after a ban was getting re-banned within seconds", - ], - }, - { - title: "Reorganize Queue Polish", - description: "cleaned up some race conditions in the queue. behavior is solid now.", - features: [ - "• worker pick + status flip is atomic now — cancel can\'t land between them and let a cancelled item still run", - "• swapped lock + wakeup-event for a single threading.Condition — newly-queued items don\'t sleep up to 60s anymore", - "• bulk enqueue dedupes within a single batch (was only deduping against pre-existing items)", - "• reorganize-preview Apply button no longer gets stuck disabled on errors", - "• db helpers let exceptions bubble instead of swallowing them as \"album not found\"", - ], - }, - { - title: "Reorganize Queue with Live Status Panel", - description: "reorganize is now a queue with a live status panel. spam-click all you want — items run one at a time and you can keep browsing.", - features: [ - "• per-album reorganize and reorganize all both enqueue into a single backend queue", - "• buttons stay clickable — clicking the same album twice silently dedupes", - "• status panel shows active progress, queued count, and recent finishes", - "• expand the panel for the full queue + per-item cancel buttons (running items can\'t be cancelled mid-flight)", - "• cross-artist items get tagged so you know what\'s queued from where", - "• continue-on-failure: one bad album never stalls the queue", - "• reorganize all is now one backend call instead of N js-driven calls — way faster", - ], - }, - { - title: "Fix Wrong-Artist Tracks Silently Downloading", - description: "searching for a track could silently download a completely different artist\'s song with the same name. fixed at two layers.", - features: [ - "• example: \"maduk — leave a light on\" on tidal was downloading tom walker\'s song of the same name with maduk\'s metadata embedded", - "• tightened the candidate artist gate (was letting through 0.4 similarity, now blocks at 0.5)", - "• acoustid verification now FAILs (quarantines) clear artist mismatches instead of accepting them", - "• ambiguous matches (covers, collabs) still get the benefit of the doubt — only obvious mismatches get blocked", - ], - }, - { - title: "Tidal Search Falls Back on Long Queries", - description: "tidal\'s search chokes on long remix-credit queries. now retries with shorter variants when the original returns 0 results.", - features: [ - "• example: \"maduk transformations remixed fire away fred v remix\" returned 0 — falls back to shorter queries until tidal finds the track", - "• up to 4 shortened variants tried, capped at 5 total requests", - "• qualifier-safe: live/remix/acoustic searches only accept fallback results that keep the qualifier", - "• returns empty if no variant preserves the qualifiers — same as before", - ], - }, - { - title: "Manual Discovery Fixes Persist Across Restart", - description: "manual discovery fixes are now saved under your active metadata source instead of always \"spotify\" — so deezer / itunes / discogs / hydrabase users\' fixes survive restart.", - features: [ - "• affects tidal, deezer, spotify public, youtube, and discovery pool manual fixes", - "• matches how the auto-discovery worker already saved", - "• spotify-primary users unaffected (hardcoded value matched their source)", - ], - }, - { - title: "Watchlist Content Filters Fixed", - description: "global override and live-version detection now behave the way the ui implies.", - features: [ - "• scheduled auto-watchlist honors watchlist → global override (was bypassing it)", - "• live detection tightened — no more false positives on titles like \"what we live for\"", - "• same fix applies to the library maintenance live/commentary cleaner", - "• still catches (live), - live, live at/from/in/on, unplugged, in concert", - ], - }, - { - title: "Discography Backfill", - description: "new maintenance job that scans each artist\'s full discography and finds what you\'re missing.", - features: [ - "• scans each library artist against your metadata source", - "• creates findings for missing tracks — review and add to wishlist", - "• respects all content filters (live, remix, acoustic, etc.) and release type filters", - "• optional auto-add-to-wishlist setting for hands-off operation", - "• opt-in, runs weekly, processes up to 50 artists per run", - ], - }, - { - title: "Repair 'Run Now' Honored While Paused", - description: "force-running a repair job no longer stalls forever when the master worker is paused.", - features: [ - "• jobs queued via run now complete even if the master worker is paused", - "• fixes silent stalls where the job logged \"scanning 50 artists\" then did nothing", - "• master-pause still blocks scheduled runs — only affects user-triggered runs", - ], - }, - { - title: "Multi-Artist Tagging", - description: "more control over how multiple artists are written to audio file tags.", - features: [ - "• configurable separator: comma, semicolon, or slash", - "• multi-value ARTISTS tag for navidrome / jellyfin multi-artist linking", - "• \"move featured artists to title\" mode — primary in ARTIST tag, others as (feat. ...) in title", - "• opt-in, defaults match current behavior", - ], - }, - { - title: "Enriched Downloads Page", - description: "download cards now show rich metadata instead of just filenames.", - features: [ - "• album artwork thumbnail on each card", - "• artist name, album name, source badge", - "• quality badge appears after post-processing", - "• falls back gracefully for transfers without metadata context", - ], - }, - { - title: "Template Variable Delimiters", - description: "use ${var} syntax to append literal text to template variables.", - features: [ - "• ${albumtype}s produces \"Albums\", \"Singles\", \"EPs\"", - "• both $var and ${var} syntaxes work everywhere", - "• validation updated to accept delimited variables", - ], - }, - { - title: "Reorganize All Albums", - description: "bulk reorganize all albums for an artist from the enhanced library view.", - features: [ - "• new reorganize all button in the artist header", - "• processes sequentially with progress toasts", - "• continues on error — one failed album doesn\'t block the rest", - "• uses the same template + endpoint as per-album reorganize", - ], - }, - { - title: "SoulSync Standalone Library", - description: "use soulsync without plex, jellyfin, or navidrome — manage your library directly.", - features: [ - "• new standalone server option in settings → connections", - "• downloads and imports write to the library db immediately", - "• pre-populated enrichment ids — workers skip re-discovery", - "• deep scan finds untracked files and removes stale db records", - "• sync page hidden automatically in standalone mode", - "• full library / artist detail / discography all work standalone", - ], - usage_note: "settings → connections → standalone. no media server needed.", - }, - { - title: "Auto-Import", - description: "background folder watcher that automatically identifies and imports music into your library.", - features: [ - "• recursive scan — any folder depth (artist/album/tracks, loose files, whatever)", - "• tag-based identification preferred, acoustid fingerprinting as fallback", - "• stats bar, filter pills, scan now, approve all, clear history", - "• expandable per-track match details with confidence scores", - "• race condition fix prevents duplicate processing on multi-track albums", - ], - usage_note: "import page → auto tab. set your import folder in settings.", - }, - { - title: "Wishlist Nebula", - description: "wishlist redesigned as an interactive artist orb visualization.", - features: [ - "• each artist is a glowing orb — albums and singles orbit around it", - "• click orbs to expand and download directly from the nebula", - "• live progress with spinning ring animation while processing", - "• stats strip up top: total artists, albums, singles, tracks", - ], - usage_note: "click wishlist in the sidebar.", - }, - { - title: "Automation Group Management", - description: "organize and manage automation groups properly.", - features: [ - "• rename, delete, and bulk-toggle groups from the group header", - "• drag-and-drop automations between groups", - "• delete confirmation shows group name and automation count", - ], - usage_note: "use the action buttons on group headers in the automations page.", - }, - { - title: "Bidirectional Artist Sync & Server Playlists", - description: "artist sync now goes both ways, and server playlists show full coverage.", - features: [ - "• artist sync pulls new content from your media server AND removes stale library entries", - "• deep scan mode fetches full metadata for newly-discovered tracks", - "• server playlist view shows all playlists with synced vs unsynced visual separation", - ], - }, - { - title: "Provider-Agnostic Discovery", - description: "discovery features work with any configured metadata source instead of requiring spotify.", - features: [ - "• similar artist matching, discovery pool, and incremental updates use source priority", - "• falls back through spotify, itunes, deezer in configured order", - "• musicmap url encoding fixed for artists with special characters", - "• freshness check simplified to age-based", - ], - }, - { - title: "Dashboard & Navigation", - description: "dashboard improvements and sidebar navigation enhancements.", - features: [ - "• library status card on dashboard — server state, track counts, scan buttons", - "• tools page in sidebar — maintenance tools moved out of the dashboard modal", - "• watchlist and wishlist promoted to full sidebar pages with live count badges", - "• acoustid scanner scans full library with retag / redownload / delete fix options", - ], - }, - { - title: "MusicBrainz & Metadata Fixes", - description: "critical tag embedding fix and picard-style album consistency.", - features: [ - "• source id tags (spotify, musicbrainz, deezer, audiodb) were silently skipped on every download — now embed correctly", - "• picard-style release preference scoring prevents navidrome album splits", - "• source tags wiped when metadata enhancement is skipped or fails", - "• spotify api no longer called when deezer/itunes is your primary source", - ], - }, - { - title: "Downloads & Soulseek Improvements", - description: "better download management, search accuracy, and queue control.", - features: [ - "• downloads batch panel — color-coded cards with progress, cancel, expand, 7-day history", - "• soulseek queries include album name now — fewer wrong-artist downloads", - "• reject results from various artists / unknown artist folders", - "• clearing wishlist cancels the active wishlist download batch", - "• album delete with \"delete files too\" option on enhanced library", - "• fix download modal freezing mid-download (m3u auto-save was exhausting server threads)", - "• fix unknown artist when adding playlist tracks to wishlist", - ], - }, - { - title: "Recent Fixes", - description: "smaller bug fixes from recent releases and community reports.", - features: [ - "• fix watchlist scan false failures — empty discography no longer reported as error", - "• fix deezer_artist_id column error on enhanced library sync", - "• fix wishlist button intermittently not navigating", - "• fix worker orb tooltips rendering behind dashboard content", - "• fix oauth callback port hardcoding — custom ports respected now", - "• fix allow duplicates and replace-lower-quality settings not saving", - "• fix wishlist dropping cross-album tracks when duplicates enabled", - "• fix spotify enrichment worker infinite loop on pre-matched artists", - "• reject qobuz 30-second sample/preview downloads", - "• auto wing-it fallback for failed discovery", - "• fix album track lookup hardcoded to spotify — uses configured primary now", - "• fix m3u showing all tracks as missing after post-processing", - "• fix acoustid retag not writing corrected tags to file", - "• fix downloads badge dropping to 300 after opening downloads page", - "• unmatch discovery tracks (red ✕ button)", - "• customizable music video naming with $artist, $title, $year", - "• fix soulseek log spam when not configured as download source", - ], - }, - { - title: "Earlier in v2.3", - description: "major features from earlier in this release cycle.", - features: [ - "• centralized downloads page with live-updating list and filter pills", - "• first-run setup wizard — 7-step guided configuration", - "• music videos — search and download from youtube", - "• inbound music request api for external tools (discord bots, home assistant)", - "• lidarr download source (in development) for usenet / torrent", - "• graceful shutdown — all workers respond to shutdown signals immediately", - "• unknown artist prevention with 3-tier metadata fallback", - "• deezer multi-artist tagging via contributors field", - "• artist map — watchlist constellation, genre map, artist explorer", - "• discogs integration — enrichment worker, fallback source, search tab", - "• wing it mode, global search bar, redesigned notifications", - "• server playlist manager, sync history dashboard, playlist explorer", - "• enhanced library manager with inline tag editing and write-to-file", - "• automation signals, multi-source search tabs, rich artist profiles", + "• reorganize queue with live status panel — bulk reorganize all, atomic status flips, race conditions fixed", + "• discography backfill maintenance job — finds what's missing across your library", + "• standalone library mode — use SoulSync without Plex, Jellyfin, or Navidrome", + "• auto-import background folder watcher — tag-based + acoustid fingerprint identification", + "• wishlist nebula — interactive artist orb visualization with inline download", + "• bidirectional artist sync + server playlist view", + "• provider-agnostic discovery — similar artists, discovery pool, and incremental updates use source priority", + "• multi-artist tagging: configurable separator, multi-value ARTISTS tag, move-to-title mode", + "• search page source icons — per-source cache with cache dots, instant source switching", + "• tidal: rejects silent quality downgrades; spotify: post-ban cooldown bumped to 30 minutes", ], }, ]; @@ -4339,7 +3906,7 @@ function _getLatestWhatsNewVersion() { const versions = Object.keys(WHATS_NEW) .filter(v => _compareVersions(v, buildVer) <= 0) .sort((a, b) => _compareVersions(b, a)); - return versions[0] || '2.5.2'; + return versions[0] || '2.6.0'; } function openWhatsNew() { diff --git a/webui/static/init.js b/webui/static/init.js index b416fb83..6b760b0c 100644 --- a/webui/static/init.js +++ b/webui/static/init.js @@ -707,8 +707,19 @@ function showPinDialog(profile) { const dialog = document.getElementById('profile-pin-dialog'); const avatar = document.getElementById('profile-pin-avatar'); const nameEl = document.getElementById('profile-pin-name'); - const input = document.getElementById('profile-pin-input'); const errorEl = document.getElementById('profile-pin-error'); + const oldInput = document.getElementById('profile-pin-input'); + const oldSubmit = document.getElementById('profile-pin-submit'); + const oldCancel = document.getElementById('profile-pin-cancel'); + + // Replace controls on every open so stale listeners from a previous + // profile cannot submit the new PIN against the old profile id. + const input = oldInput.cloneNode(true); + const submit = oldSubmit.cloneNode(true); + const cancel = oldCancel.cloneNode(true); + oldInput.parentNode.replaceChild(input, oldInput); + oldSubmit.parentNode.replaceChild(submit, oldSubmit); + oldCancel.parentNode.replaceChild(cancel, oldCancel); renderProfileAvatar(avatar, profile); nameEl.textContent = profile.name; @@ -718,13 +729,12 @@ function showPinDialog(profile) { dialog.style.display = 'flex'; setTimeout(() => input.focus(), 100); - const submit = document.getElementById('profile-pin-submit'); - const cancel = document.getElementById('profile-pin-cancel'); - const wasSwitching = !!currentProfile; const handleSubmit = async () => { const pin = input.value; if (!pin) return; + submit.disabled = true; + submit.textContent = 'Verifying...'; try { const res = await fetch('/api/profiles/select', { method: 'POST', @@ -753,7 +763,8 @@ function showPinDialog(profile) { errorEl.textContent = 'Connection error'; errorEl.style.display = ''; } - cleanup(); + submit.disabled = false; + submit.textContent = 'Submit'; }; const handleCancel = () => { @@ -2125,15 +2136,9 @@ function initApp() { // =============================== function initializeNavigation() { - const navButtons = document.querySelectorAll('.nav-button'); - - navButtons.forEach(button => { - button.addEventListener('click', () => { - const page = button.getAttribute('data-page'); - navigateToPage(page); - }); - }); - + // Sidebar navigation is now driven by native link navigation. + // Page activation and active-state styling are synchronized from the + // current URL by the shell bridge and route controllers. } const _DEEPLINK_VALID_PAGES = new Set([ @@ -2150,14 +2155,41 @@ function _getPageFromPath() { const path = window.location.pathname.replace(/^\/+|\/+$/g, ''); if (!path) return 'dashboard'; - const basePage = path.split('/')[0]; + const segs = path.split('/'); + const basePage = segs[0]; if (!_DEEPLINK_VALID_PAGES.has(basePage)) return 'dashboard'; // Context-dependent pages fall back to a sensible parent - if (basePage === 'artist-detail') return 'library'; if (basePage === 'playlist-explorer') return 'library'; return basePage; } +function _normalizeArtistDetailSource(source) { + const value = (source || '').toString().trim().toLowerCase(); + return value || 'library'; +} + +function buildArtistDetailPath(artistId, source = null) { + if (!artistId) { + throw new Error('artistId is required for artist-detail navigation'); + } + const normalizedSource = _normalizeArtistDetailSource(source); + return '/artist-detail/' + encodeURIComponent(normalizedSource) + '/' + encodeURIComponent(String(artistId)); +} + +function parseArtistDetailPath(pathname = window.location.pathname) { + const segs = String(pathname || '').split('/').filter(Boolean); + if (segs[0] !== 'artist-detail' || segs.length < 3) return null; + + const source = decodeURIComponent(segs[1] || ''); + const artistId = decodeURIComponent(segs.slice(2).join('/')); + if (!source || !artistId) return null; + + return { + artistId, + source: source.toLowerCase() === 'library' ? null : source, + }; +} + // =============================== // MOBILE NAVIGATION // =============================== @@ -2271,6 +2303,10 @@ function navigateToPage(pageId, options = {}) { return; } + if (pageId === 'artist-detail' && !options.artistId) { + return false; + } + const router = getWebRouter(); if (router && !options.skipRouteChange) { notifyPageWillChange(pageId); @@ -2279,7 +2315,11 @@ function navigateToPage(pageId, options = {}) { showReactHost(pageId); setActivePageChrome(pageId); } - return router.navigateToPage(pageId, { replace: options.replace === true }); + return router.navigateToPage(pageId, { + replace: options.replace === true, + artistId: options.artistId, + artistSource: options.artistSource, + }); } // Fallback path for initial bootstrap or environments without TanStack routing. @@ -2294,7 +2334,9 @@ function navigateToPage(pageId, options = {}) { } if (!options.skipPushState) { - const urlPath = pageId === 'dashboard' ? '/' : '/' + pageId; + const urlPath = pageId === 'dashboard' ? '/' + : (pageId === 'artist-detail' && options.artistId) ? buildArtistDetailPath(options.artistId, options.artistSource) + : '/' + pageId; if (window.location.pathname !== urlPath) { if (options.replace === true) { history.replaceState({ page: pageId }, '', urlPath); @@ -2341,7 +2383,10 @@ async function loadPageData(pageId) { case 'library': // Check if we should return to artist detail view instead of list if (artistDetailPageState.currentArtistId && artistDetailPageState.currentArtistName) { - navigateToPage('artist-detail'); + navigateToPage('artist-detail', { + artistId: artistDetailPageState.currentArtistId, + artistSource: artistDetailPageState.currentArtistSource, + }); if (!artistDetailPageState.isInitialized) { initializeArtistDetailPage(); loadArtistDetailData(artistDetailPageState.currentArtistId, artistDetailPageState.currentArtistName); @@ -2355,7 +2400,7 @@ async function loadPageData(pageId) { } break; case 'artist-detail': - // Artist detail page is handled separately by navigateToArtistDetail() + // Artist detail page is entered through the route handoff and legacy navigator. break; case 'discover': if (!discoverPageInitialized) { @@ -2375,9 +2420,6 @@ async function loadPageData(pageId) { loadApiKeys(); loadBlacklistCount(); break; - case 'stats': - initializeStatsPage(); - break; case 'import': initializeImportPage(); break; @@ -2426,3 +2468,164 @@ async function loadPageData(pageId) { showToast(`Failed to load ${pageId} data`, 'error'); } } + +// ---- Dashboard cursor-following accent blob (two-layer liquid) ---- +// Both layers lerp toward a target point: the cursor when it's hovering +// any .dash-card, otherwise the grid center (idle resting position). +// Core layer (--blob-x/y) follows faster, halo (--blob-x-soft/y-soft) +// trails. Each card renders both layers and clips them to its own bounds +// via overflow:hidden, so the blob spans the bento while gaps stay dark. +// Disabled entirely when body.reduce-effects is set. +(function initDashboardCursorBlob() { + let grid = null; + let cards = []; + let cardRects = []; // cached rects, refreshed each frame + let targetX = 0, targetY = 0; + let coreX = 0, coreY = 0; + let softX = 0, softY = 0; + let rafId = 0; + let attached = false; + let centeredOnce = false; + + const RECENTER_DELAY_MS = 1500; + let recenterTimer = 0; + + const isReduced = () => document.body.classList.contains('reduce-effects'); + + const gridCenter = () => { + const r = grid.getBoundingClientRect(); + return { x: r.left + r.width / 2, y: r.top + r.height / 2 }; + }; + + // Two-pass per frame: read all rects first (one layout flush), then + // write all CSS vars (no further reads). Avoids per-card layout thrash. + const tick = () => { + if (isReduced()) { rafId = 0; return; } + + coreX += (targetX - coreX) * 0.040; + coreY += (targetY - coreY) * 0.040; + softX += (targetX - softX) * 0.022; + softY += (targetY - softY) * 0.022; + + const n = cards.length; + if (cardRects.length !== n) cardRects.length = n; + for (let i = 0; i < n; i++) cardRects[i] = cards[i].getBoundingClientRect(); + for (let i = 0; i < n; i++) { + const r = cardRects[i]; + const s = cards[i].style; + s.setProperty('--blob-x', (coreX - r.left) + 'px'); + s.setProperty('--blob-y', (coreY - r.top) + 'px'); + s.setProperty('--blob-x-soft', (softX - r.left) + 'px'); + s.setProperty('--blob-y-soft', (softY - r.top) + 'px'); + } + + const dx = Math.abs(targetX - softX) + Math.abs(targetX - coreX); + const dy = Math.abs(targetY - softY) + Math.abs(targetY - coreY); + if (dx + dy > 0.4) rafId = requestAnimationFrame(tick); + else rafId = 0; + }; + + const ensureLoop = () => { + if (!rafId && !isReduced()) rafId = requestAnimationFrame(tick); + }; + + const cancelRecenter = () => { + if (recenterTimer) { clearTimeout(recenterTimer); recenterTimer = 0; } + }; + const recenterNow = () => { + recenterTimer = 0; + if (!grid) return; + const c = gridCenter(); + targetX = c.x; targetY = c.y; + ensureLoop(); + }; + const scheduleRecenter = () => { + if (recenterTimer) return; + recenterTimer = setTimeout(recenterNow, RECENTER_DELAY_MS); + }; + + // Snap the blob to grid center the first time the grid becomes + // measurable (page may not be visible at DOMContentLoaded). + const snapToCenterIfReady = () => { + if (!grid || centeredOnce) return; + const r = grid.getBoundingClientRect(); + if (r.width === 0 || r.height === 0) return; // not visible yet + const c = { x: r.left + r.width / 2, y: r.top + r.height / 2 }; + targetX = coreX = softX = c.x; + targetY = coreY = softY = c.y; + centeredOnce = true; + ensureLoop(); + }; + + function attach() { + if (attached) return; + grid = document.querySelector('.dash-grid'); + if (!grid) return; + attached = true; + cards = Array.from(grid.querySelectorAll('.dash-card')); + + snapToCenterIfReady(); + + grid.addEventListener('pointermove', (e) => { + if (isReduced()) return; + const onCard = e.target && e.target.closest && e.target.closest('.dash-card'); + if (onCard) { + cancelRecenter(); + targetX = e.clientX; + targetY = e.clientY; + ensureLoop(); + } else { + scheduleRecenter(); + } + }); + grid.addEventListener('pointerleave', () => { + if (!isReduced()) scheduleRecenter(); + }); + window.addEventListener('resize', () => { + if (isReduced()) return; + // Idle: snap immediately. Active: respect the existing delay. + if (!recenterTimer) recenterNow(); + }); + + // Re-resolve cards when active-downloads card toggles visibility. + const cardObserver = new MutationObserver(() => { + cards = Array.from(grid.querySelectorAll('.dash-card')); + ensureLoop(); + }); + cardObserver.observe(grid, { childList: true, subtree: false, attributes: true, attributeFilter: ['style', 'class'] }); + + // If the grid was hidden at attach time, snap once it becomes + // measurable (page navigation, tab switch). + if (!centeredOnce && 'IntersectionObserver' in window) { + const visObserver = new IntersectionObserver((entries) => { + for (const ent of entries) { + if (ent.isIntersecting) { snapToCenterIfReady(); break; } + } + }); + visObserver.observe(grid); + } + + // React to reduce-effects toggle on body class. + const bodyObserver = new MutationObserver(() => { + if (isReduced()) { + cancelRecenter(); + if (rafId) { cancelAnimationFrame(rafId); rafId = 0; } + } else { + centeredOnce = false; + snapToCenterIfReady(); + } + }); + bodyObserver.observe(document.body, { attributes: true, attributeFilter: ['class'] }); + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', attach); + } else { + attach(); + } + // Also retry on full load — covers late-mounted markup. + window.addEventListener('load', () => { + attach(); + snapToCenterIfReady(); + }); +})(); diff --git a/webui/static/library.js b/webui/static/library.js index cacc2ba6..7a4cae2a 100644 --- a/webui/static/library.js +++ b/webui/static/library.js @@ -218,6 +218,7 @@ function displayLibraryArtists(artists) { // Ignore clicks on badge icons (they open external links / toggle watchlist) const badge = e.target.closest('.source-card-icon'); if (badge) { + e.preventDefault(); e.stopPropagation(); const url = badge.dataset.url; if (url) { window.open(url, '_blank'); return; } @@ -233,10 +234,6 @@ function displayLibraryArtists(artists) { } return; } - const card = e.target.closest('.library-artist-card'); - if (card) { - navigateToArtistDetail(card.dataset.artistId, card.dataset.artistName); - } }; } @@ -259,6 +256,7 @@ function buildLibraryArtistCardHTML(artist, index) { if (artist.tidal_id) badges.push({ logo: TIDAL_LOGO_URL, fb: 'TD', title: 'Tidal', url: `https://tidal.com/browse/artist/${artist.tidal_id}` }); if (artist.qobuz_id) badges.push({ logo: QOBUZ_LOGO_URL, fb: 'Qz', title: 'Qobuz', url: `https://www.qobuz.com/artist/${artist.qobuz_id}` }); if (artist.discogs_id) badges.push({ logo: DISCOGS_LOGO_URL, fb: 'DC', title: 'Discogs', url: `https://www.discogs.com/artist/${artist.discogs_id}` }); + if (artist.amazon_id) badges.push({ logo: AMAZON_LOGO_URL, fb: 'AMZ', title: 'Amazon Music', url: null }); if (artist.soul_id && !String(artist.soul_id).startsWith('soul_unnamed_')) badges.push({ logo: '/static/trans2.png', fb: 'SS', title: `SoulID: ${artist.soul_id}`, url: null }); // Watchlist badge @@ -298,14 +296,14 @@ function buildLibraryArtistCardHTML(artist, index) { // Track stats const trackStat = artist.track_count > 0 ? `${artist.track_count} track${artist.track_count !== 1 ? 's' : ''}` : ''; - return `
+ return ` ${badgeContainerHTML} ${imageHTML}

${_esc(artist.name)}

${trackStat}
-
`; + `; } function updateLibraryPagination(pagination) { @@ -682,16 +680,31 @@ async function toggleLibraryCardWatchlist(btn, artist) { // =============================================== // Artist detail page state +const _ARTIST_DETAIL_BACK_LABELS = { + library: 'Back to Library', + search: 'Back to Search', + discover: 'Back to Discover', + watchlist: 'Back to Watchlist', + wishlist: 'Back to Wishlist', + stats: 'Back to Stats', + 'playlist-explorer': 'Back to Explorer', + automations: 'Back to Automations', + dashboard: 'Back to Dashboard', + sync: 'Back to Sync', + 'active-downloads': 'Back to Downloads', +}; + +// Stack of origins for the back-button label. Each entry: {type:'page', pageId} +// or {type:'artist', name}. Pushed on forward navigation, popped on back. +// Separate from browser history — only used for the label display. +let _artistDetailLabelStack = []; +let _artistDetailGoingBack = false; + let artistDetailPageState = { isInitialized: false, currentArtistId: null, currentArtistName: null, currentArtistSource: null, - // Stack of origins captured by navigateToArtistDetail for the back button. - // Each entry is either {type:'page', pageId} or {type:'artist', id, name, source} - // so chained navigation (Search → A → similar B → similar C) walks back one - // step at a time instead of jumping straight to Search. - originStack: [], enhancedView: false, enhancedData: null, expandedAlbums: new Set(), @@ -709,7 +722,6 @@ function clearArtistDetailPageState() { artistDetailPageState.currentArtistId = null; artistDetailPageState.currentArtistName = null; artistDetailPageState.currentArtistSource = null; - artistDetailPageState.originStack = []; } if (typeof window !== 'undefined') { @@ -785,59 +797,43 @@ if (typeof window !== 'undefined') { } -// Friendly labels for the dynamic "← Back to X" button on the artist-detail page. -// Page id (the value of currentPage) -> button label. -const _ARTIST_DETAIL_BACK_LABELS = { - library: 'Back to Library', - search: 'Back to Search', - discover: 'Back to Discover', - watchlist: 'Back to Watchlist', - wishlist: 'Back to Wishlist', - stats: 'Back to Stats', - 'playlist-explorer': 'Back to Explorer', - automations: 'Back to Automations', - dashboard: 'Back to Dashboard', - sync: 'Back to Sync', - 'active-downloads': 'Back to Downloads', -}; - function navigateToArtistDetail(artistId, artistName, sourceOverride = null, options = {}) { + const normalizedSource = sourceOverride || null; + + // Skip reload if already on this exact artist/source (prevents double-fetch + // when the router fires activateLegacyPath after navigating to an + // /artist-detail/:source/:id URL). + if (artistId && + String(artistId) === String(artistDetailPageState.currentArtistId) && + String(normalizedSource || '') === String(artistDetailPageState.currentArtistSource || '')) { + if (currentPage !== 'artist-detail') { + navigateToPage('artist-detail', { + artistId, + artistSource: normalizedSource, + skipRouteChange: options.skipRouteChange === true + }); + } + return; + } console.log(`🎵 Navigating to artist detail: ${artistName} (ID: ${artistId}${sourceOverride ? `, source: ${sourceOverride}` : ''})`); - // Capture the current location on the origin stack BEFORE navigateToPage - // flips currentPage. The back button walks this stack one step at a time, - // so a chain like Search → A → similar B → similar C steps back through - // C → B → A → Search instead of jumping straight home. `skipOriginPush` - // lets the back button re-enter a prior artist without re-pushing. - if (!options.skipOriginPush) { - // Fresh entry (from a non-artist page) starts a new chain; any stale - // entries from a prior artist-detail visit are dropped. + // Maintain the label stack. Back navigations pop; forward navigations push. + // Only treat the flag as a back-nav signal when we're still on artist-detail — + // if history.back() landed on a non-artist page first, the flag is stale. + if (_artistDetailGoingBack && currentPage === 'artist-detail') { + _artistDetailLabelStack.pop(); + _artistDetailGoingBack = false; + } else { + _artistDetailGoingBack = false; // clear any stale flag if (currentPage !== 'artist-detail') { - artistDetailPageState.originStack = []; + _artistDetailLabelStack = []; // fresh chain from a non-artist page } - - let entry; - if (currentPage === 'artist-detail' && artistDetailPageState.currentArtistId) { - entry = { - type: 'artist', - id: artistDetailPageState.currentArtistId, - name: artistDetailPageState.currentArtistName, - source: artistDetailPageState.currentArtistSource, - }; + if (currentPage === 'artist-detail' && artistDetailPageState.currentArtistName) { + _artistDetailLabelStack.push({ type: 'artist', name: artistDetailPageState.currentArtistName }); } else { const pageId = (typeof currentPage === 'string' && currentPage && currentPage !== 'artist-detail') ? currentPage : 'library'; - entry = { type: 'page', pageId }; - } - - // Avoid pushing a duplicate top entry on repeated clicks of the same target. - const top = artistDetailPageState.originStack[artistDetailPageState.originStack.length - 1]; - const isDuplicate = top && top.type === entry.type && ( - (entry.type === 'page' && top.pageId === entry.pageId) || - (entry.type === 'artist' && String(top.id) === String(entry.id)) - ); - if (!isDuplicate) { - artistDetailPageState.originStack.push(entry); + _artistDetailLabelStack.push({ type: 'page', pageId }); } } @@ -855,7 +851,7 @@ function navigateToArtistDetail(artistId, artistName, sourceOverride = null, opt // Store current artist info and reset enhanced view state artistDetailPageState.currentArtistId = artistId; artistDetailPageState.currentArtistName = artistName; - artistDetailPageState.currentArtistSource = sourceOverride || null; + artistDetailPageState.currentArtistSource = normalizedSource; artistDetailPageState.enhancedData = null; artistDetailPageState.expandedAlbums = new Set(); artistDetailPageState.selectedTracks = new Set(); @@ -884,16 +880,19 @@ function navigateToArtistDetail(artistId, artistName, sourceOverride = null, opt if (bulkBar) bulkBar.classList.remove('visible'); // Navigate to artist detail page - navigateToPage('artist-detail'); - - // Update back-button label to reflect where the next pop will land. - _updateArtistDetailBackButtonLabel(); + navigateToPage('artist-detail', { + artistId, + artistSource: normalizedSource, + skipRouteChange: options.skipRouteChange === true + }); // Initialize if needed and load data if (!artistDetailPageState.isInitialized) { initializeArtistDetailPage(); } + _updateArtistDetailBackButtonLabel(); + // Load artist data loadArtistDetailData(artistId, artistName); } @@ -901,11 +900,12 @@ function navigateToArtistDetail(artistId, artistName, sourceOverride = null, opt function _updateArtistDetailBackButtonLabel() { const backBtnLabel = document.querySelector('#artist-detail-back-btn span'); if (!backBtnLabel) return; - const stack = artistDetailPageState.originStack || []; - const top = stack[stack.length - 1]; + const top = _artistDetailLabelStack[_artistDetailLabelStack.length - 1]; if (!top) { - backBtnLabel.textContent = `← ${_ARTIST_DETAIL_BACK_LABELS.library}`; - } else if (top.type === 'artist') { + backBtnLabel.textContent = '← Back'; + return; + } + if (top.type === 'artist') { backBtnLabel.textContent = `← Back to ${top.name}`; } else { const friendly = _ARTIST_DETAIL_BACK_LABELS[top.pageId] || _ARTIST_DETAIL_BACK_LABELS.library; @@ -916,9 +916,8 @@ function _updateArtistDetailBackButtonLabel() { function initializeArtistDetailPage() { console.log("🔧 Initializing Artist Detail page..."); - // Initialize back button — pops the origin stack one step at a time so a - // chain like Search → A → B → C walks back through C → B → A → Search - // instead of jumping straight to the original entry page. + // Initialize back button — use browser history when possible, with a + // simple library fallback if the user lands here without in-app history. const backBtn = document.getElementById("artist-detail-back-btn"); if (backBtn) { backBtn.addEventListener("click", () => { @@ -928,28 +927,16 @@ function initializeArtistDetailPage() { artistDetailPageState.completionController = null; } - const stack = artistDetailPageState.originStack || []; - if (stack.length > 0) { - const target = stack.pop(); - if (target.type === 'artist') { - // Re-enter a prior artist in the chain without re-pushing, - // so the stack keeps shrinking as the user steps back. - navigateToArtistDetail(target.id, target.name, target.source, { skipOriginPush: true }); - return; - } - // target.type === 'page' — fully exit the artist-detail chain - artistDetailPageState.currentArtistId = null; - artistDetailPageState.currentArtistName = null; - artistDetailPageState.originStack = []; - navigateToPage(target.pageId); + if (window.history.length > 1) { + _artistDetailGoingBack = true; + window.history.back(); return; } - // No history — default to library - artistDetailPageState.currentArtistId = null; - artistDetailPageState.currentArtistName = null; - artistDetailPageState.originStack = []; - navigateToPage('library'); + // No history — fall back to recorded origin page or library + const top = _artistDetailLabelStack.pop(); + _updateArtistDetailBackButtonLabel(); + navigateToPage(top?.type === 'page' ? (top.pageId || 'library') : 'library'); }); } @@ -1019,6 +1006,17 @@ async function loadArtistDetailData(artistId, artistName) { throw new Error(data.error || 'Failed to load artist data'); } + const isSourceOnlyArtist = !data.artist?.server_source; + if (isSourceOnlyArtist && data.discography) { + for (const bucket of ['albums', 'eps', 'singles']) { + for (const release of (data.discography[bucket] || [])) { + if (release.owned === null || typeof release.owned === 'undefined') { + release.owned = false; + } + } + } + } + console.log(`✅ Loaded artist detail data:`, data); // Hide loading and show all content @@ -1044,6 +1042,15 @@ async function loadArtistDetailData(artistId, artistName) { artistDetailPageState.currentArtistId = data.artist.id; } + // Backfill name from API response — URL-driven navigation passes '' for the + // name so the label stack has real names when the user clicks a similar artist. + if (data.artist?.name && !artistDetailPageState.currentArtistName) { + artistDetailPageState.currentArtistName = data.artist.name; + if (typeof _updateSidebarLibraryBreadcrumb === 'function') { + _updateSidebarLibraryBreadcrumb(); + } + } + // Keep the resolved metadata source for album-track lookups. artistDetailPageState.currentArtistSource = data.discography?.source || data.artist?.source || null; @@ -1054,7 +1061,7 @@ async function loadArtistDetailData(artistId, artistName) { renderArtistEnrichmentCoverage(data.enrichment_coverage); // Start streaming ownership checks if we have Spotify discography with checking state - if (data.discography && data.discography.albums) { + if (!isSourceOnlyArtist && data.discography && data.discography.albums) { const hasChecking = [...(data.discography.albums || []), ...(data.discography.eps || []), ...(data.discography.singles || [])] .some(r => r.owned === null); if (hasChecking) { @@ -1068,7 +1075,9 @@ async function loadArtistDetailData(artistId, artistName) { // Use currentArtistId (not the closure arg) because the library-upgrade // branch above may have rewritten it from the source ID to the library PK, // and /api/library/artist//quality-analysis only works on library PKs. - checkArtistEnhanceEligibility(artistDetailPageState.currentArtistId); + if (!isSourceOnlyArtist) { + checkArtistEnhanceEligibility(artistDetailPageState.currentArtistId); + } } catch (error) { console.error(`❌ Error loading artist detail data:`, error); @@ -1128,6 +1137,7 @@ function updateArtistDetailPageHeaderWithData(artist) { if (artist.tidal_id) badges.push(_hb(TIDAL_LOGO_URL, 'TD', 'Tidal', `https://tidal.com/browse/artist/${artist.tidal_id}`)); if (artist.qobuz_id) badges.push(_hb(QOBUZ_LOGO_URL, 'Qz', 'Qobuz', `https://www.qobuz.com/artist/${artist.qobuz_id}`)); if (artist.discogs_id) badges.push(_hb(DISCOGS_LOGO_URL, 'DC', 'Discogs', `https://www.discogs.com/artist/${artist.discogs_id}`)); + if (artist.amazon_id) badges.push(_hb(AMAZON_LOGO_URL, 'AMZ', 'Amazon Music', null)); if (artist.soul_id && !String(artist.soul_id).startsWith('soul_unnamed_')) badges.push(_hb('/static/trans2.png', 'SS', `SoulID: ${artist.soul_id}`, null)); badgesContainer.innerHTML = badges.join(''); @@ -1228,6 +1238,9 @@ function populateArtistDetailPage(data) { // MusicMap name lookup). Fire-and-forget — the function handles its own // loading state and errors. if (artist && artist.name && typeof loadSimilarArtists === 'function') { + if (typeof cancelSimilarArtistsLoad === 'function') { + cancelSimilarArtistsLoad(); + } loadSimilarArtists(artist.name); } } @@ -1312,15 +1325,34 @@ function updateArtistHeaderStats(albumCount, trackCount) { console.log("📊 Using new hero section instead of old header stats"); } +function _isUsableArtistHeroImageUrl(url) { + return typeof url === 'string' && url.trim() !== '' && url !== 'null'; +} + +function _getArtistHeroReleaseImage(discography) { + for (const bucket of ['albums', 'eps', 'singles']) { + for (const release of (discography?.[bucket] || [])) { + if (_isUsableArtistHeroImageUrl(release?.image_url)) { + return release.image_url; + } + } + } + return ''; +} + function updateArtistHeroSection(artist, discography) { console.log("🖼️ Updating artist hero section"); + const artistImageUrl = _isUsableArtistHeroImageUrl(artist.image_url) ? artist.image_url : ''; + const releaseImageUrl = _getArtistHeroReleaseImage(discography); + const primaryHeroImageUrl = artistImageUrl || releaseImageUrl; + // Blurred background image (inline-Artists hero treatment) — set whenever // we have an image_url; falls back to clearing the bg if not. const heroBg = document.getElementById("artist-detail-hero-bg"); if (heroBg) { - if (artist.image_url && artist.image_url.trim() !== "" && artist.image_url !== "null") { - heroBg.style.backgroundImage = `url('${artist.image_url}')`; + if (primaryHeroImageUrl) { + heroBg.style.backgroundImage = `url('${primaryHeroImageUrl}')`; } else { heroBg.style.backgroundImage = ''; } @@ -1337,9 +1369,11 @@ function updateArtistHeroSection(artist, discography) { console.log(` - Image element:`, imageElement); console.log(` - Fallback element:`, fallbackElement); - if (artist.image_url && artist.image_url.trim() !== "" && artist.image_url !== "null") { - console.log(`✅ Setting image src to: ${artist.image_url}`); - imageElement.src = artist.image_url; + if (primaryHeroImageUrl) { + console.log(`✅ Setting image src to: ${primaryHeroImageUrl}`); + imageElement.dataset.triedDeezer = ''; + imageElement.dataset.triedReleaseFallback = artistImageUrl ? '' : 'true'; + imageElement.src = primaryHeroImageUrl; imageElement.alt = artist.name; imageElement.style.display = "block"; if (fallbackElement) { @@ -1351,11 +1385,14 @@ function updateArtistHeroSection(artist, discography) { }; imageElement.onerror = () => { - console.error(`❌ Failed to load artist image: ${artist.image_url}`); - // Try Deezer fallback before emoji + console.error(`❌ Failed to load artist image: ${imageElement.src}`); + // Try Deezer fallback, then release art, before the generic icon. if (artist.deezer_id && !imageElement.dataset.triedDeezer) { imageElement.dataset.triedDeezer = 'true'; imageElement.src = `https://api.deezer.com/artist/${artist.deezer_id}/image?size=big`; + } else if (releaseImageUrl && imageElement.src !== releaseImageUrl && !imageElement.dataset.triedReleaseFallback) { + imageElement.dataset.triedReleaseFallback = 'true'; + imageElement.src = releaseImageUrl; } else { imageElement.style.display = "none"; if (fallbackElement) { @@ -1475,6 +1512,70 @@ const _TOP_TRACKS_SOURCE_LABELS = { lastfm: 'Popular on Last.fm', }; +async function playTrackByMetadata(title, artist, album = '') { + // 1. Try the library first — fastest and best quality if owned. + try { + const resp = await fetch('/api/stats/resolve-track', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ title, artist }), + }); + const data = await resp.json(); + if (data.success && data.track) { + const track = data.track; + playLibraryTrack( + { + id: track.id, + title: track.title, + file_path: track.file_path, + bitrate: track.bitrate, + artist_id: track.artist_id, + album_id: track.album_id, + _stats_image: track.image_url || null, + }, + track.album_title || album || '', + track.artist_name || artist || '', + ); + return; + } + } catch (e) { + console.debug('Library resolve failed, will try streaming fallback:', e); + } + + // 2. Library miss — fall back to streaming via the enhanced-search streamer. + if (typeof showLoadingOverlay === 'function') { + showLoadingOverlay(`Searching for ${title}...`); + } + try { + const streamResp = await fetch('/api/enhanced-search/stream-track', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + track_name: title, + artist_name: artist, + album_name: album, + duration_ms: 0, + }), + }); + const streamData = await streamResp.json(); + if (typeof hideLoadingOverlay === 'function') hideLoadingOverlay(); + + if (streamData.success && streamData.result) { + if (typeof startStream === 'function') { + await startStream(streamData.result); + } else { + showToast('Streaming not available', 'error'); + } + } else { + showToast(streamData.error || 'Track not found in library or any source', 'error'); + } + } catch (e) { + if (typeof hideLoadingOverlay === 'function') hideLoadingOverlay(); + showToast('Failed to play track', 'error'); + console.error('Stream fallback failed:', e); + } +} + async function _loadArtistTopTracks(artistName) { const sidebar = document.getElementById('artist-hero-sidebar'); const container = document.getElementById('hero-top-tracks'); @@ -1539,7 +1640,7 @@ async function _loadArtistTopTracks(artistName) { const playBtn = e.target.closest('.hero-top-track-play'); if (playBtn) { e.stopPropagation(); - playStatsTrack(playBtn.dataset.track, playBtn.dataset.artist, ''); + playTrackByMetadata(playBtn.dataset.track, playBtn.dataset.artist, ''); return; } const dlBtn = e.target.closest('.hero-top-track-download'); @@ -1595,7 +1696,7 @@ async function _loadArtistTopTracks(artistName) { const btn = e.target.closest('.hero-top-track-play'); if (btn) { e.stopPropagation(); - playStatsTrack(btn.dataset.track, btn.dataset.artist, ''); + playTrackByMetadata(btn.dataset.track, btn.dataset.artist, ''); } }; sidebar.style.display = ''; @@ -2959,6 +3060,7 @@ function renderArtistMetaPanel(artist) { { key: 'genius_url', label: 'Genius', svc: 'genius' }, { key: 'tidal_id', label: 'Tidal', svc: 'tidal' }, { key: 'qobuz_id', label: 'Qobuz', svc: 'qobuz' }, + { key: 'amazon_id', label: 'Amazon Music', svc: 'amazon' }, ]; idSources.forEach(src => { if (artist[src.key]) { @@ -3094,6 +3196,7 @@ function renderArtistMetaPanel(artist) { { key: 'genius_match_status', label: 'Genius', attempted: 'genius_last_attempted', svc: 'genius' }, { key: 'tidal_match_status', label: 'Tidal', attempted: 'tidal_last_attempted', svc: 'tidal' }, { key: 'qobuz_match_status', label: 'Qobuz', attempted: 'qobuz_last_attempted', svc: 'qobuz' }, + { key: 'amazon_match_status', label: 'Amazon', attempted: 'amazon_last_attempted', svc: 'amazon' }, ]; statusServices.forEach(s => { const status = artist[s.key]; @@ -3354,11 +3457,27 @@ function toggleAlbumExpand(albumId) { inner.appendChild(renderAlbumMetaRow(album)); inner.appendChild(renderTrackTable(album)); inner.dataset.rendered = 'true'; + ensureEnhancedAlbumCanonicalTracks(album).then(updated => { + if (!updated || !artistDetailPageState.expandedAlbums.has(albumId)) return; + rerenderEnhancedAlbumPanel(album.id); + }); } } } } +function rerenderEnhancedAlbumPanel(albumId) { + const panel = document.getElementById(`enhanced-tracks-panel-${albumId}`); + const inner = panel?.querySelector('.enhanced-tracks-panel-inner'); + const album = findEnhancedAlbum(albumId); + if (!inner || !album) return; + inner.innerHTML = ''; + inner.appendChild(renderExpandedAlbumHeader(album)); + inner.appendChild(renderAlbumMetaRow(album)); + inner.appendChild(renderTrackTable(album)); + inner.dataset.rendered = 'true'; +} + function findEnhancedAlbum(albumId) { // Use cached map for O(1) lookups instead of O(n) array scan if (artistDetailPageState._albumMap) { @@ -3404,8 +3523,21 @@ function renderExpandedAlbumHeader(album) { const details = []; if (album.year) details.push(String(album.year)); - const trackCount = album.tracks ? album.tracks.length : 0; - details.push(`${trackCount} track${trackCount !== 1 ? 's' : ''}`); + const ownedTrackCount = album.tracks ? album.tracks.length : 0; + const visibleTrackRows = _getEnhancedAlbumTrackRows(album); + const expectedTrackCount = Math.max( + ownedTrackCount, + visibleTrackRows.length, + Number(album.api_track_count || album.track_count || 0) + ); + const missingCount = visibleTrackRows.filter(t => t._missingExpected).length; + if (album._canonicalTracksLoading) details.push('checking tracklist'); + if (expectedTrackCount > ownedTrackCount) { + details.push(`${ownedTrackCount}/${expectedTrackCount} tracks`); + } else { + details.push(`${ownedTrackCount} track${ownedTrackCount !== 1 ? 's' : ''}`); + } + if (missingCount > 0) details.push(`${missingCount} missing`); let durMs = 0; (album.tracks || []).forEach(t => { durMs += (t.duration || 0); }); if (durMs > 0) details.push(formatDurationMs(durMs)); @@ -3462,6 +3594,7 @@ function renderExpandedAlbumHeader(album) { { key: 'discogs_match_status', label: 'Discogs', attempted: 'discogs_last_attempted', svc: 'discogs' }, { key: 'itunes_match_status', label: 'iTunes', attempted: 'itunes_last_attempted', svc: 'itunes' }, { key: 'lastfm_match_status', label: 'Last.fm', attempted: 'lastfm_last_attempted', svc: 'lastfm' }, + { key: 'amazon_match_status', label: 'Amazon', attempted: 'amazon_last_attempted', svc: 'amazon' }, ]; statusSvcs.forEach(s => { const status = album[s.key]; @@ -3639,20 +3772,167 @@ function renderAlbumMetaRow(album) { return row; } +function _trackSlotKey(track) { + const disc = Number(track.disc_number || track.expected_disc_number || 1); + const num = Number(track.track_number || track.expected_track_number || 0); + return `${disc}:${num}`; +} + +function _normalizeExpectedMissingTrack(source, album) { + const title = source.title || source.name || `Track ${source.track_number || '?'}`; + const sourceTrackId = source.track_id || source.id || source.source_track_id || ''; + const hasActionableContext = !!( + title && + source.track_number && + (sourceTrackId || source.spotify_track_id || source.deezer_id || source.itunes_track_id || source.musicbrainz_recording_id) + ); + return { + id: `missing-${album.id}-${source.disc_number || 1}-${source.track_number || ''}`, + title, + track_number: source.track_number || source.position || '', + disc_number: source.disc_number || 1, + duration: source.duration || source.duration_ms || 0, + spotify_track_id: source.spotify_track_id || (source.source === 'spotify' ? sourceTrackId : ''), + deezer_id: source.deezer_id || (source.source === 'deezer' ? sourceTrackId : ''), + itunes_track_id: source.itunes_track_id || (source.source === 'itunes' ? sourceTrackId : ''), + musicbrainz_recording_id: source.musicbrainz_recording_id || (source.source === 'musicbrainz' ? sourceTrackId : ''), + source: source.source || source.metadata_source || '', + track_id: sourceTrackId, + album_id: source.album_id || source.source_album_id || '', + artists: source.artists || source.artist_names || [], + _hasActionableContext: hasActionableContext, + _missingExpected: true, + _sourceTrack: source, + }; +} + +function _getEnhancedAlbumCanonicalSource(album) { + const priority = [ + ['spotify', 'spotify_album_id'], + ['deezer', 'deezer_id'], + ['itunes', 'itunes_album_id'], + ['musicbrainz', 'musicbrainz_release_id'], + ['discogs', 'discogs_id'], + ['tidal', 'tidal_id'], + ['qobuz', 'qobuz_id'], + ]; + for (const [source, key] of priority) { + if (album[key]) return { source, id: album[key] }; + } + return null; +} + +async function ensureEnhancedAlbumCanonicalTracks(album) { + if (!album || album._canonicalTracksLoaded || album._canonicalTracksLoading) return false; + + const canonicalSource = _getEnhancedAlbumCanonicalSource(album); + if (!canonicalSource) { + album._canonicalTracksLoaded = true; + return false; + } + + album._canonicalTracksLoading = true; + try { + const artistName = artistDetailPageState.enhancedData?.artist?.name || artistDetailPageState.currentArtistName || ''; + const params = new URLSearchParams({ + name: album.title || '', + artist: artistName, + source: canonicalSource.source, + }); + const response = await fetch(`/api/album/${encodeURIComponent(canonicalSource.id)}/tracks?${params}`); + const data = await response.json(); + if (!response.ok || !data.success) { + throw new Error(data.error || 'Failed to load canonical tracklist'); + } + + const canonicalTracks = Array.isArray(data.tracks) ? data.tracks : []; + album.canonical_tracks = canonicalTracks.map((track, index) => ({ + ...track, + title: track.title || track.name || `Track ${track.track_number || index + 1}`, + name: track.name || track.title || `Track ${track.track_number || index + 1}`, + track_number: track.track_number || index + 1, + disc_number: track.disc_number || 1, + duration: track.duration || track.duration_ms || 0, + source: data.source || canonicalSource.source, + track_id: track.id || track.track_id || '', + id: track.id || track.track_id || `${canonicalSource.source}:${canonicalSource.id}:${track.disc_number || 1}:${track.track_number || index + 1}`, + })); + album.api_track_count = Math.max(Number(album.api_track_count || 0), album.canonical_tracks.length); + album.missing_tracks = _deriveEnhancedMissingTracks(album, album.canonical_tracks); + album._canonicalTracksLoaded = true; + return true; + } catch (error) { + album._canonicalTracksError = error.message; + album._canonicalTracksLoaded = true; + console.debug('Failed to load canonical album tracks:', album.title, error); + return false; + } finally { + album._canonicalTracksLoading = false; + } +} + +function _deriveEnhancedMissingTracks(album, canonicalTracks) { + const occupiedSlots = new Set(); + (album.tracks || []).forEach(track => { + const key = _trackSlotKey(track); + if (key !== '1:0') occupiedSlots.add(key); + }); + return (canonicalTracks || []) + .map(track => ({ + ...track, + name: track.name || track.title, + duration_ms: track.duration_ms || track.duration || 0, + })) + .filter(track => { + const key = _trackSlotKey(track); + const normalized = _normalizeExpectedMissingTrack(track, album); + return key !== '1:0' && normalized._hasActionableContext && !occupiedSlots.has(key); + }); +} + +function _getEnhancedAlbumTrackRows(album) { + const ownedTracks = Array.isArray(album.tracks) ? album.tracks : []; + const rowsBySlot = new Map(); + ownedTracks.forEach(track => { + const key = _trackSlotKey(track); + rowsBySlot.set(key === '1:0' ? `owned:${track.id}` : key, track); + }); + + const explicitMissing = Array.isArray(album.missing_tracks) ? album.missing_tracks : []; + explicitMissing.forEach(missing => { + const row = _normalizeExpectedMissingTrack(missing, album); + const key = _trackSlotKey(row); + if (row._hasActionableContext && !rowsBySlot.has(key)) rowsBySlot.set(key, row); + }); + + return Array.from(rowsBySlot.values()).sort((a, b) => { + const discDelta = Number(a.disc_number || 1) - Number(b.disc_number || 1); + if (discDelta !== 0) return discDelta; + const trackDelta = Number(a.track_number || 0) - Number(b.track_number || 0); + if (trackDelta !== 0) return trackDelta; + return String(a.title || '').localeCompare(String(b.title || '')); + }); +} + function _buildTrackRow(track, album, admin) { const tr = document.createElement('tr'); tr.dataset.trackId = track.id; tr.dataset.albumId = album.id; + tr._enhancedTrack = track; + tr._enhancedAlbum = album; + if (track._missingExpected) tr.classList.add('enhanced-missing-track-row'); if (artistDetailPageState.selectedTracks.has(String(track.id))) tr.classList.add('selected'); // Checkbox (admin only) if (admin) { const cbTd = document.createElement('td'); - const cb = document.createElement('input'); - cb.type = 'checkbox'; - cb.className = 'enhanced-track-checkbox'; - cb.checked = artistDetailPageState.selectedTracks.has(String(track.id)); - cbTd.appendChild(cb); + if (!track._missingExpected) { + const cb = document.createElement('input'); + cb.type = 'checkbox'; + cb.className = 'enhanced-track-checkbox'; + cb.checked = artistDetailPageState.selectedTracks.has(String(track.id)); + cbTd.appendChild(cb); + } tr.appendChild(cbTd); } @@ -3661,7 +3941,7 @@ function _buildTrackRow(track, album, admin) { playTd.className = 'col-play'; const playBtn = document.createElement('button'); playBtn.className = 'enhanced-play-btn'; - playBtn.innerHTML = '▶'; + playBtn.innerHTML = track._missingExpected ? '—' : '▶'; playBtn.title = track.file_path ? 'Play track' : 'No file available'; if (!track.file_path) playBtn.disabled = true; playTd.appendChild(playBtn); @@ -3683,6 +3963,13 @@ function _buildTrackRow(track, album, admin) { const titleTd = document.createElement('td'); titleTd.className = 'col-title' + (admin ? ' editable' : ''); titleTd.textContent = track.title || 'Unknown'; + if (track._missingExpected) { + titleTd.classList.remove('editable'); + const status = document.createElement('span'); + status.className = 'enhanced-missing-track-badge'; + status.textContent = 'Missing'; + titleTd.appendChild(status); + } tr.appendChild(titleTd); // Duration @@ -3694,12 +3981,16 @@ function _buildTrackRow(track, album, admin) { // Format const fmtTd = document.createElement('td'); fmtTd.className = 'col-format'; - const format = extractFormat(track.file_path); - const fmtSpan = document.createElement('span'); - const fmtClass = format === 'FLAC' ? 'flac' : (format === 'MP3' ? 'mp3' : 'other'); - fmtSpan.className = `enhanced-format-badge ${fmtClass}`; - fmtSpan.textContent = format; - fmtTd.appendChild(fmtSpan); + if (track._missingExpected) { + fmtTd.textContent = '-'; + } else { + const format = extractFormat(track.file_path); + const fmtSpan = document.createElement('span'); + const fmtClass = format === 'FLAC' ? 'flac' : (format === 'MP3' ? 'mp3' : 'other'); + fmtSpan.className = `enhanced-format-badge ${fmtClass}`; + fmtSpan.textContent = format; + fmtTd.appendChild(fmtSpan); + } tr.appendChild(fmtTd); // Bitrate @@ -3722,7 +4013,9 @@ function _buildTrackRow(track, album, admin) { const pathTd = document.createElement('td'); pathTd.className = 'col-path'; const filePath = track.file_path || '-'; - const fileName = filePath !== '-' ? filePath.split(/[\\/]/).pop() : '-'; + const fileName = track._missingExpected + ? 'Missing from library' + : (filePath !== '-' ? filePath.split(/[\\/]/).pop() : '-'); pathTd.textContent = fileName; pathTd.title = filePath; tr.appendChild(pathTd); @@ -3756,7 +4049,7 @@ function _buildTrackRow(track, album, admin) { // Add to Queue button const queueTd = document.createElement('td'); queueTd.className = 'col-queue'; - if (track.file_path) { + if (!track._missingExpected && track.file_path) { const queueBtn = document.createElement('button'); queueBtn.className = 'enhanced-queue-btn'; queueBtn.innerHTML = '+'; @@ -3769,7 +4062,7 @@ function _buildTrackRow(track, album, admin) { // Write Tags button (admin only) const tagTd = document.createElement('td'); tagTd.className = 'col-writetag'; - if (track.file_path) { + if (track.file_path && !track._missingExpected) { const tagBtn = document.createElement('button'); tagBtn.className = 'enhanced-write-tag-btn'; tagBtn.innerHTML = '✎'; @@ -3784,26 +4077,42 @@ function _buildTrackRow(track, album, admin) { } tr.appendChild(tagTd); - // Track actions cell — source info, redownload, delete (admin only) + // Track actions cell: source info, redownload, delete, or missing-track actions. const actionsTd = document.createElement('td'); actionsTd.className = 'col-track-actions'; - actionsTd.innerHTML = ` -
- - - -
- `; + if (track._missingExpected) { + actionsTd.innerHTML = ` +
+ +
+ `; + } else { + actionsTd.innerHTML = ` +
+ + + +
+ `; + } tr.appendChild(actionsTd); } else { // Report Issue button per track (non-admin) const reportTd = document.createElement('td'); reportTd.className = 'col-report'; - const reportBtn = document.createElement('button'); - reportBtn.className = 'enhanced-track-report-btn'; - reportBtn.innerHTML = '⚑'; - reportBtn.title = 'Report issue with this track'; - reportTd.appendChild(reportBtn); + if (track._missingExpected) { + const manageBtn = document.createElement('button'); + manageBtn.className = 'enhanced-missing-manage-btn'; + manageBtn.textContent = 'Manage'; + manageBtn.dataset.action = 'manage-missing'; + reportTd.appendChild(manageBtn); + } else { + const reportBtn = document.createElement('button'); + reportBtn.className = 'enhanced-track-report-btn'; + reportBtn.innerHTML = '⚑'; + reportBtn.title = 'Report issue with this track'; + reportTd.appendChild(reportBtn); + } tr.appendChild(reportTd); } @@ -3821,6 +4130,14 @@ function _buildTrackRow(track, album, admin) { } function _getTrackDataFromRow(tr) { + if (tr._enhancedTrack && tr._enhancedAlbum) { + return { + track: tr._enhancedTrack, + album: tr._enhancedAlbum, + trackId: tr._enhancedTrack.id, + albumId: tr._enhancedAlbum.id + }; + } const trackId = tr.dataset.trackId; const albumId = tr.dataset.albumId; const album = findEnhancedAlbum(albumId); @@ -3867,6 +4184,13 @@ function _attachTableDelegation(table, album) { if (!info) return; const { track, trackId } = info; + const manageAction = target.closest('.enhanced-missing-manage-btn'); + if (manageAction && track._missingExpected) { + e.stopPropagation(); + openMissingTrackManageModal(track, album); + return; + } + // Checkbox if (target.classList.contains('enhanced-track-checkbox')) { toggleTrackSelection(String(trackId)); @@ -4060,7 +4384,7 @@ function _rebuildTbody(table, album) { const admin = isEnhancedAdmin(); const oldTbody = table.querySelector('tbody'); const newTbody = document.createElement('tbody'); - (album.tracks || []).forEach(track => { + _getEnhancedAlbumTrackRows(album).forEach(track => { newTbody.appendChild(_buildTrackRow(track, album, admin)); }); if (oldTbody) table.replaceChild(newTbody, oldTbody); @@ -4069,13 +4393,13 @@ function _rebuildTbody(table, album) { function renderTrackTable(album) { const wrapper = document.createElement('div'); - const tracks = album.tracks || []; // Re-apply stored sort order if any const activeSort = artistDetailPageState.enhancedTrackSort[album.id]; if (activeSort) { sortEnhancedTracks(album, activeSort.field, activeSort.ascending); } + const tracks = _getEnhancedAlbumTrackRows(album); if (tracks.length === 0) { wrapper.innerHTML = '
No tracks in database
'; @@ -4335,8 +4659,8 @@ async function showTrackSourceInfo(track, anchorEl) { return; } - const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer: '💜' }; - const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer: 'Deezer' }; + const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer: '💜', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', auto_import: '📥', staging: '📥', torrent: '🧲', usenet: '📰' }; + const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer: 'Deezer', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', auto_import: 'Auto-Import', staging: 'Staging', torrent: 'Torrent', usenet: 'Usenet' }; const dl = data.downloads[0]; // Most recent download const icon = serviceIcons[dl.source_service] || '📦'; @@ -4666,8 +4990,8 @@ async function _streamRedownloadSources(overlay, track, metadata) { const startBtn = document.getElementById('redownload-start-btn'); if (!columnsEl) return; - const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡' }; - const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto' }; + const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', torrent: '🧲', usenet: '📰' }; + const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet' }; let allCandidates = []; let firstResult = true; @@ -4789,8 +5113,8 @@ async function _streamRedownloadSources(overlay, track, metadata) { /* _renderRedownloadStep2 removed — replaced by _streamRedownloadSources above */ if (false) { - const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡' }; - const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto' }; + const serviceIcons = { soulseek: '🔍', youtube: '▶️', tidal: '🌊', qobuz: '🎵', hifi: '🎧', deezer_dl: '💜', hybrid: '⚡', lidarr: '📦', amazon: '🛒', soundcloud: '☁️', torrent: '🧲', usenet: '📰' }; + const serviceLabels = { soulseek: 'Soulseek', youtube: 'YouTube', tidal: 'Tidal', qobuz: 'Qobuz', hifi: 'HiFi', deezer_dl: 'Deezer', hybrid: 'Auto', lidarr: 'Lidarr', amazon: 'Amazon Music', soundcloud: 'SoundCloud', torrent: 'Torrent', usenet: 'Usenet' }; // Group candidates by source service const grouped = {}; @@ -5125,6 +5449,14 @@ function getServiceUrl(service, entityType, id) { album: `https://www.qobuz.com/album/${id}`, track: `https://www.qobuz.com/track/${id}`, }, + discogs: { + artist: `https://www.discogs.com/artist/${id}`, + album: `https://www.discogs.com/release/${id}`, + }, + amazon: { + album: `https://music.amazon.com/albums/${id}`, + track: `https://music.amazon.com/tracks/${id}`, + }, }; return urls[service] && urls[service][entityType] || null; } @@ -5564,7 +5896,8 @@ function openManualMatchModal(entityType, entityId, service, defaultQuery, artis const serviceLabels = { spotify: 'Spotify', musicbrainz: 'MusicBrainz', deezer: 'Deezer', - audiodb: 'AudioDB', itunes: 'iTunes', lastfm: 'Last.fm', genius: 'Genius' + audiodb: 'AudioDB', itunes: 'iTunes', lastfm: 'Last.fm', genius: 'Genius', + tidal: 'Tidal', qobuz: 'Qobuz', amazon: 'Amazon Music' }; const overlay = document.createElement('div'); @@ -5771,6 +6104,397 @@ async function applyManualMatch(entityType, entityId, service, serviceId, artist } } +async function wishlistEnhancedMissingTrack(track, album, downloadNow = false) { + if (!track._hasActionableContext) { + showToast('This missing track needs metadata context before it can be wishlisted or downloaded.', 'error'); + return; + } + const artistName = artistDetailPageState.enhancedData?.artist?.name || artistDetailPageState.currentArtistName || ''; + const artist = { + id: artistDetailPageState.enhancedData?.artist?.id || artistDetailPageState.currentArtistId || '', + name: artistName, + image_url: artistDetailPageState.enhancedData?.artist?.thumb_url || getArtistImageFromPage() || '', + }; + const albumData = { + id: album.id, + name: album.title || 'Unknown Album', + title: album.title || 'Unknown Album', + image_url: album.thumb_url || '', + release_date: album.year ? `${album.year}-01-01` : '', + album_type: album.record_type || 'album', + total_tracks: Number(album.api_track_count || album.track_count || album.tracks?.length || 1), + }; + const wishlistTrack = { + id: track.spotify_track_id || track.deezer_id || track.itunes_track_id || track.musicbrainz_recording_id || track.id, + name: track.title || `Track ${track.track_number || ''}`, + title: track.title || `Track ${track.track_number || ''}`, + artists: [{ name: artistName }], + duration_ms: track.duration || 0, + track_number: track.track_number || 1, + disc_number: track.disc_number || 1, + album: albumData, + }; + + if (typeof openAddToWishlistModal !== 'function') { + showToast('Wishlist modal is not available on this page', 'error'); + return; + } + + await openAddToWishlistModal(albumData, artist, [wishlistTrack], albumData.album_type, { [wishlistTrack.name]: false }); + if (downloadNow && typeof handleWishlistDownloadNow === 'function') { + setTimeout(() => handleWishlistDownloadNow(), 150); + } +} + +function openMissingTrackManageModal(track, album) { + if (track._missingExpected && !track._hasActionableContext) { + showToast('This missing track needs metadata context before it can be managed.', 'error'); + return; + } + + const existing = document.getElementById('enhanced-missing-manage-overlay'); + if (existing) existing.remove(); + + const artistName = artistDetailPageState.enhancedData?.artist?.name || artistDetailPageState.currentArtistName || ''; + const overlay = document.createElement('div'); + overlay.id = 'enhanced-missing-manage-overlay'; + overlay.className = 'modal-overlay'; + let isImporting = false; + overlay.onclick = (e) => { if (e.target === overlay && !isImporting) overlay.remove(); }; + + const modal = document.createElement('div'); + modal.className = 'confirm-modal enhanced-missing-manage-modal'; + modal.innerHTML = ` +
+

Manage Missing Track

+ +
+
+
+
Missing album slot
+
#${escapeHtml(String(track.track_number || '?'))} ${escapeHtml(track.title || 'Unknown Track')}
+
${escapeHtml(artistName)} · ${escapeHtml(album.title || '')}
+
+
+ + +
+
+
+ +
+ `; + + overlay.appendChild(modal); + document.body.appendChild(overlay); + + const close = () => overlay.remove(); + modal.querySelector('.confirm-modal-close').onclick = close; + modal.querySelector('[data-action="cancel"]').onclick = close; + modal.querySelectorAll('.enhanced-missing-option').forEach(button => { + button.onclick = async () => { + const action = button.dataset.action; + close(); + if (action === 'library') { + await wishlistEnhancedMissingTrack(track, album, false); + } else if (action === 'have') { + openHaveMissingTrackModal(track, album); + } + }; + }); +} + +function openHaveMissingTrackModal(track, album) { + if (track._missingExpected && !track._hasActionableContext) { + showToast('This missing track needs metadata context before it can be imported.', 'error'); + return; + } + const existing = document.getElementById('enhanced-have-track-overlay'); + if (existing) existing.remove(); + + const artistName = artistDetailPageState.enhancedData?.artist?.name || artistDetailPageState.currentArtistName || ''; + const overlay = document.createElement('div'); + overlay.id = 'enhanced-have-track-overlay'; + overlay.className = 'modal-overlay'; + let isImporting = false; + overlay.onclick = (e) => { if (e.target === overlay && !isImporting) overlay.remove(); }; + + const modal = document.createElement('div'); + modal.className = 'enhanced-manual-match-modal enhanced-have-track-modal'; + modal.innerHTML = ` +
+
+

I Have This Track

+
Use an existing file as the source audio. SoulSync will copy it into this album.
+
+ +
+
+
Missing album slot
+
#${escapeHtml(String(track.track_number || '?'))} ${escapeHtml(track.title || 'Unknown Track')}
+
${escapeHtml(artistName)} · ${escapeHtml(album.title || '')}
+
+
+ + +
+
+
Searching your library...
+
+ +
The selected file stays in its current album/folder. SoulSync copies it, writes the missing track's tags, and places the copy in this album.
+ +
+ + +
+ `; + overlay.appendChild(modal); + document.body.appendChild(overlay); + + let selectedTrackId = null; + let selectedTrackSummary = ''; + let importTimer = null; + const searchResultsById = new Map(); + const searchInput = modal.querySelector('#enhanced-have-track-search'); + const resultsEl = modal.querySelector('#enhanced-have-track-results'); + const selectedEl = modal.querySelector('#enhanced-have-selected'); + const selectedText = selectedEl.querySelector('strong'); + const confirmBtn = modal.querySelector('#enhanced-have-confirm'); + const cancelBtn = modal.querySelector('#enhanced-have-cancel'); + const closeBtn = modal.querySelector('.enhanced-bulk-modal-close'); + const searchBtn = modal.querySelector('#enhanced-have-track-search-btn'); + const statusEl = modal.querySelector('#enhanced-have-import-status'); + const statusTitle = statusEl.querySelector('.enhanced-have-import-title'); + const statusDetail = statusEl.querySelector('.enhanced-have-import-detail'); + const statusTime = statusEl.querySelector('.enhanced-have-import-time'); + + const close = () => { + if (isImporting) return; + if (importTimer) clearInterval(importTimer); + overlay.remove(); + }; + closeBtn.onclick = close; + cancelBtn.onclick = close; + searchBtn.onclick = () => runSearch(); + searchInput.onkeydown = (e) => { if (e.key === 'Enter' && !isImporting) runSearch(); }; + + function selectResultRow(row) { + if (isImporting || !row) return; + const trackId = row.dataset.trackId; + const result = searchResultsById.get(String(trackId)); + if (!trackId || !result) return; + resultsEl.querySelectorAll('.enhanced-have-result-row').forEach(r => { + r.classList.remove('selected'); + r.setAttribute('aria-pressed', 'false'); + }); + row.classList.add('selected'); + row.setAttribute('aria-pressed', 'true'); + selectedTrackId = trackId; + selectedTrackSummary = `${result.title || 'Unknown'}${result.album_title ? ` from ${result.album_title}` : ''}`; + selectedText.textContent = selectedTrackSummary; + selectedEl.hidden = false; + confirmBtn.disabled = false; + } + + resultsEl.addEventListener('click', (event) => { + const row = event.target.closest('.enhanced-have-result-row'); + if (!row || !resultsEl.contains(row)) return; + event.preventDefault(); + selectResultRow(row); + }); + + resultsEl.addEventListener('keydown', (event) => { + if (event.key !== 'Enter' && event.key !== ' ') return; + const row = event.target.closest('.enhanced-have-result-row'); + if (!row) return; + event.preventDefault(); + selectResultRow(row); + }); + + function setImportStatus(title, detail, tone = 'working') { + statusEl.hidden = false; + statusEl.classList.toggle('error', tone === 'error'); + statusEl.classList.toggle('success', tone === 'success'); + statusTitle.textContent = title; + statusDetail.textContent = detail; + } + + function startImportTimer() { + const start = Date.now(); + const stages = [ + { after: 0, text: 'Copying selected file into staging.' }, + { after: 4, text: 'Verifying audio and writing the missing track tags.' }, + { after: 10, text: 'Post-processing can take a moment for FLAC files, lyrics, ReplayGain, and metadata.' }, + { after: 20, text: 'Still working. Waiting for the backend to finish and return the refreshed library row.' }, + ]; + if (importTimer) clearInterval(importTimer); + importTimer = setInterval(() => { + const elapsed = Math.floor((Date.now() - start) / 1000); + statusTime.textContent = `${elapsed}s`; + const stage = [...stages].reverse().find(item => elapsed >= item.after); + if (stage) statusDetail.textContent = stage.text; + }, 250); + } + + async function runSearch() { + const query = searchInput.value.trim(); + if (!query) { + resultsEl.innerHTML = '
Enter a title or artist to search.
'; + return; + } + selectedTrackId = null; + selectedTrackSummary = ''; + selectedEl.hidden = true; + confirmBtn.disabled = true; + resultsEl.innerHTML = '
Searching...
'; + searchResultsById.clear(); + try { + const res = await fetch(`/api/library/search-tracks?q=${encodeURIComponent(query)}&limit=12`); + const data = await res.json(); + if (!data.success) throw new Error(data.error || 'Search failed'); + const tracks = data.tracks || []; + if (tracks.length === 0) { + resultsEl.innerHTML = '
No library tracks found. Try a different search.
'; + return; + } + resultsEl.innerHTML = ''; + tracks.forEach(result => { + if (!result.id) return; + searchResultsById.set(String(result.id), result); + const row = document.createElement('div'); + row.className = 'enhanced-have-result-row'; + row.dataset.trackId = String(result.id); + row.setAttribute('role', 'button'); + row.setAttribute('tabindex', '0'); + row.setAttribute('aria-pressed', 'false'); + const fileName = result.file_path ? result.file_path.split(/[\\/]/).pop() : 'No file path'; + row.innerHTML = ` + + + ${escapeHtml(result.title || 'Unknown')} + ${escapeHtml(result.artist_name || '')}${result.album_title ? ` · ${escapeHtml(result.album_title)}` : ''} + ${escapeHtml(fileName)} + + + ${result.duration ? `${formatDurationMs(result.duration)}` : ''} + ${result.bitrate ? `${result.bitrate} kbps` : ''} + + `; + resultsEl.appendChild(row); + }); + } catch (error) { + resultsEl.innerHTML = `
Error: ${escapeHtml(error.message)}
`; + } + } + + confirmBtn.onclick = async () => { + if (!selectedTrackId) return; + isImporting = true; + confirmBtn.disabled = true; + confirmBtn.textContent = 'Importing...'; + cancelBtn.disabled = true; + closeBtn.disabled = true; + searchBtn.disabled = true; + searchInput.disabled = true; + resultsEl.querySelectorAll('.enhanced-have-result-row').forEach(row => { + row.setAttribute('aria-disabled', 'true'); + row.classList.add('disabled'); + }); + setImportStatus( + 'Importing selected file', + selectedTrackSummary ? `Using ${selectedTrackSummary}.` : 'Using the selected library track.' + ); + startImportTimer(); + try { + const sourceTrack = track._sourceTrack || track; + const expectedTrack = { + title: track.title || sourceTrack.title || sourceTrack.name || '', + name: track.title || sourceTrack.title || sourceTrack.name || '', + track_number: track.track_number || sourceTrack.track_number, + disc_number: track.disc_number || sourceTrack.disc_number || 1, + duration: track.duration || sourceTrack.duration || sourceTrack.duration_ms || 0, + duration_ms: track.duration || sourceTrack.duration_ms || sourceTrack.duration || 0, + source: track.source || sourceTrack.source || '', + track_id: track.track_id || sourceTrack.track_id || sourceTrack.id || '', + id: track.track_id || sourceTrack.track_id || sourceTrack.id || '', + album_id: track.album_id || sourceTrack.album_id || '', + spotify_track_id: track.spotify_track_id || sourceTrack.spotify_track_id || '', + deezer_id: track.deezer_id || sourceTrack.deezer_id || '', + itunes_track_id: track.itunes_track_id || sourceTrack.itunes_track_id || '', + musicbrainz_recording_id: track.musicbrainz_recording_id || sourceTrack.musicbrainz_recording_id || '', + artists: track.artists || sourceTrack.artists || [artistName], + }; + const discs = (album.canonical_tracks || album.tracks || []).map(t => Number(t.disc_number || 1)); + const res = await fetch(`/api/library/album/${album.id}/import-existing-track`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + source_track_id: selectedTrackId, + expected_track: expectedTrack, + album_source_id: album.spotify_album_id || album.deezer_id || album.itunes_album_id || album.musicbrainz_release_id || album.discogs_id || album.tidal_id || album.qobuz_id || '', + total_discs: Math.max(1, ...discs), + }), + }); + setImportStatus('Finalizing import', 'Backend finished. Refreshing the enhanced library view...'); + const data = await res.json(); + if (!data.success) throw new Error(data.error || 'Failed to import track'); + if (importTimer) clearInterval(importTimer); + statusTime.textContent = statusTime.textContent || 'done'; + setImportStatus('Import complete', 'The copied file is now being shown in this album.', 'success'); + showToast('Track imported. Original file was left untouched.', 'success'); + if (data.updated_data && data.updated_data.success) { + artistDetailPageState.enhancedData = data.updated_data; + _rebuildAlbumMap(); + renderEnhancedView(); + } else if (artistDetailPageState.currentArtistId) { + await loadEnhancedViewData(artistDetailPageState.currentArtistId); + } + setTimeout(() => overlay.remove(), 650); + } catch (error) { + if (importTimer) clearInterval(importTimer); + isImporting = false; + confirmBtn.disabled = false; + confirmBtn.textContent = 'Import Track'; + cancelBtn.disabled = false; + closeBtn.disabled = false; + searchBtn.disabled = false; + searchInput.disabled = false; + resultsEl.querySelectorAll('.enhanced-have-result-row').forEach(row => { + row.setAttribute('aria-disabled', 'false'); + row.classList.remove('disabled'); + }); + setImportStatus('Import failed', error.message, 'error'); + showToast(`Import failed: ${error.message}`, 'error'); + } + }; + + searchInput.focus(); + runSearch(); +} + // ---- Enrichment ---- let _enrichmentInFlight = false; @@ -6418,10 +7142,24 @@ async function showReorganizeModal(albumId) { let html = '
'; + // Metadata MODE picker — API call (default) vs read embedded tags. + // Tag-mode (#592) trusts the user's enriched library and issues + // zero API calls. + html += '
'; + html += ''; + html += '
"API" queries your metadata source for the canonical tracklist. "Embedded tags" reads each file\'s own tags as the source of truth — useful for well-tagged libraries and avoids API calls.
'; + html += ''; + html += '
'; + // Metadata source picker — populated from /reorganize/sources. // Empty value = use configured primary (with fallback chain). // Specific source = strict mode, that source only. - html += '
'; + // Hidden when mode = 'tags' since the source picker is irrelevant + // (tags are read straight off the file). + html += '
'; html += ''; html += '
Pick which source to read the album\'s tracklist from. Defaults to your configured primary. Reorganize uses your global download template, same as fresh downloads.
'; html += ''; + html += ''; + html += ''; + html += ''; + html += '
'; + + // Source picker — applies to ALL albums in this run. Hidden when + // mode = 'tags'. Albums without an ID for the chosen source will + // be skipped at the backend with a clear status. Auto = use + // configured primary with fallback chain. + html += '
'; html += ''; html += '
Pick which source to read tracklists from. Albums without an ID for that source will be skipped. Reorganize uses your global download template, same as fresh downloads.
'; html += ' +
+

Type to search

+
+
+
+ 🎵 + Library Track +
+
+ +
+

Type to search

+
+
+ +
+
+ Existing Matches + +
+

Loading…

+
+
+ +
+
+ +
+
+ + +
+
+
+ `; + + document.body.appendChild(overlay); + _mlmOverlay = overlay; + _mlmSelectedSource = null; + _mlmSelectedLibrary = null; + _mlmUpdateSaveBtn(); + _mlmLoadMatches(); + + if (prefill) { + const src = document.getElementById('mlm-source-search'); + if (src) { src.value = prefill; _mlmSourceSearch(prefill); } + } +} + +function _mlmClose() { + if (_mlmOverlay) { _mlmOverlay.remove(); _mlmOverlay = null; } + _mlmSelectedSource = null; + _mlmSelectedLibrary = null; +} + +function _mlmSourceDebounce(q) { + clearTimeout(_mlmSourceTimer); + _mlmSourceTimer = setTimeout(() => _mlmSourceSearch(q), 300); +} +function _mlmLibraryDebounce(q) { + clearTimeout(_mlmLibraryTimer); + _mlmLibraryTimer = setTimeout(() => _mlmLibrarySearch(q), 300); +} + +async function _mlmSourceSearch(q) { + const el = document.getElementById('mlm-source-results'); + if (!el) return; + if (!q.trim()) { el.innerHTML = '

Type to search

'; return; } + el.innerHTML = '

Searching…

'; + try { + const res = await fetch(`/api/manual-library-matches/source-search?q=${encodeURIComponent(q)}&limit=15`); + const data = await res.json(); + _mlmRenderSourceResults(data.tracks || []); + } catch (e) { el.innerHTML = '

Search failed

'; } +} + +async function _mlmLibrarySearch(q) { + const el = document.getElementById('mlm-library-results'); + if (!el) return; + if (!q.trim()) { el.innerHTML = '

Type to search

'; return; } + el.innerHTML = '

Searching…

'; + try { + const res = await fetch(`/api/manual-library-matches/library-search?q=${encodeURIComponent(q)}&limit=15`); + const data = await res.json(); + _mlmRenderLibraryResults(data.tracks || []); + } catch (e) { el.innerHTML = '

Search failed

'; } +} + +function _mlmEsc(str) { + return String(str || '').replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); +} + +function _mlmRenderSourceResults(tracks) { + const el = document.getElementById('mlm-source-results'); + if (!el) return; + if (!tracks.length) { el.innerHTML = '

No results

'; return; } + el.innerHTML = tracks.map((t, i) => { + const sel = _mlmSelectedSource && _mlmSelectedSource.source_track_id === t.source_track_id ? 'mlm-row-selected' : ''; + return `
+
${_mlmEsc(t.title || '—')}
+
${_mlmEsc(t.artist || '')}${t.album ? ' · ' + _mlmEsc(t.album) : ''}
+
${_mlmEsc(t.context || t.source || '')}
+
`; + }).join(''); + el._mlmTracks = tracks; +} + +function _mlmRenderLibraryResults(tracks) { + const el = document.getElementById('mlm-library-results'); + if (!el) return; + if (!tracks.length) { el.innerHTML = '

No results

'; return; } + el.innerHTML = tracks.map((t, i) => { + const sel = _mlmSelectedLibrary && _mlmSelectedLibrary.id === t.id ? 'mlm-row-selected' : ''; + const path = t.file_path ? t.file_path.split(/[/\\]/).pop() : ''; + return `
+
${_mlmEsc(t.title || '—')}
+
${_mlmEsc(t.artist_name || '')}${t.album_title ? ' · ' + _mlmEsc(t.album_title) : ''}
+
${_mlmEsc(path)}${t.bitrate ? ' · ' + t.bitrate + 'kbps' : ''}
+
`; + }).join(''); + el._mlmTracks = tracks; +} + +function _mlmSelectSource(idx) { + const el = document.getElementById('mlm-source-results'); + if (!el || !el._mlmTracks) return; + _mlmSelectedSource = el._mlmTracks[idx]; + el.querySelectorAll('.mlm-result-row').forEach((r, i) => r.classList.toggle('mlm-row-selected', i === idx)); + _mlmUpdateSaveBtn(); +} + +function _mlmSelectLibrary(idx) { + const el = document.getElementById('mlm-library-results'); + if (!el || !el._mlmTracks) return; + _mlmSelectedLibrary = el._mlmTracks[idx]; + el.querySelectorAll('.mlm-result-row').forEach((r, i) => r.classList.toggle('mlm-row-selected', i === idx)); + _mlmUpdateSaveBtn(); +} + +function _mlmUpdateSaveBtn() { + const btn = document.getElementById('mlm-save-btn'); + if (btn) btn.disabled = !(_mlmSelectedSource && _mlmSelectedLibrary); +} + +async function _mlmSaveMatch() { + if (!_mlmSelectedSource || !_mlmSelectedLibrary) return; + const status = document.getElementById('mlm-status'); + if (status) status.textContent = 'Saving…'; + try { + const body = { + source: _mlmSelectedSource.source, + source_track_id: _mlmSelectedSource.source_track_id, + library_track_id: _mlmSelectedLibrary.id, + source_title: _mlmSelectedSource.title || '', + source_artist: _mlmSelectedSource.artist || '', + source_album: _mlmSelectedSource.album || '', + source_context_json: '', + server_source: '', + }; + const res = await fetch('/api/manual-library-matches', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + const data = await res.json(); + if (data.success) { + if (status) status.textContent = 'Saved!'; + _mlmSelectedSource = null; + _mlmSelectedLibrary = null; + _mlmUpdateSaveBtn(); + await _mlmLoadMatches(); + setTimeout(() => { if (status) status.textContent = ''; }, 2000); + } else { + if (status) status.textContent = 'Error: ' + (data.error || 'unknown'); + } + } catch (e) { + if (status) status.textContent = 'Network error'; + } +} + +async function _mlmLoadMatches() { + const el = document.getElementById('mlm-matches-list'); + if (!el) return; + try { + const res = await fetch('/api/manual-library-matches'); + const data = await res.json(); + const matches = data.matches || []; + const countEl = document.getElementById('mlm-match-count'); + if (countEl) countEl.textContent = matches.length; + if (!matches.length) { + el.innerHTML = '

No matches saved yet

'; + return; + } + el.innerHTML = ` + + ${matches.map(m => ` + + + + + `).join('')} +
Source TrackLibrary TrackSource
${_mlmEsc(m.source_title || m.source_track_id)}
${_mlmEsc(m.source_artist || '')}
${_mlmEsc(m.library_title || String(m.library_track_id))}
${_mlmEsc(m.library_artist || '')}
${_mlmEsc(m.source)}
`; + } catch (e) { + el.innerHTML = '

Failed to load matches

'; + } +} + +async function _mlmDeleteMatch(id) { + try { + await fetch(`/api/manual-library-matches/${id}`, { method: 'DELETE' }); + await _mlmLoadMatches(); + } catch (e) { + if (typeof showToast === 'function') showToast('Failed to remove match', 'error'); + } +} + // ================================= diff --git a/webui/static/media-player.js b/webui/static/media-player.js index 85f25796..fd7eb526 100644 --- a/webui/static/media-player.js +++ b/webui/static/media-player.js @@ -64,8 +64,20 @@ function toggleMediaPlayerExpansion() { function extractTrackTitle(filename) { if (!filename) return null; + // Strip the ``||`` prefix used by YouTube / + // Tidal / Qobuz / torrent / usenet plugins to thread the source- + // side identifier through ``filename`` without polluting the + // display string. The id always comes first, the human title + // after. If no separator is present, fall through with the raw + // value so existing slskd / streaming-source paths are untouched. + let title = filename; + const sepIdx = title.indexOf('||'); + if (sepIdx >= 0) { + title = title.slice(sepIdx + 2); + } + // Remove file extension - let title = filename.replace(/\.[^/.]+$/, ''); + title = title.replace(/\.[^/.]+$/, ''); // Remove path components, keep only the filename title = title.split('/').pop().split('\\').pop(); @@ -82,17 +94,28 @@ function extractTrackTitle(filename) { return title || null; } +function _stripSourceIdPrefix(value) { + // Defensive cleanup for callers that pass a raw ``||`` + // string straight into setTrackInfo without first running + // extractTrackTitle. The id always precedes the separator; the display + // string follows. Strings with no separator pass through unchanged. + if (!value || typeof value !== 'string') return value; + const idx = value.indexOf('||'); + if (idx < 0) return value; + return value.slice(idx + 2); +} + function setTrackInfo(track) { currentTrack = track; const trackTitleElement = document.getElementById('track-title'); - const trackTitle = track.title || 'Unknown Track'; + const trackTitle = _stripSourceIdPrefix(track.title) || 'Unknown Track'; // Set up the HTML structure for scrolling trackTitleElement.innerHTML = `${escapeHtml(trackTitle)}`; - document.getElementById('artist-name').textContent = track.artist || 'Unknown Artist'; - document.getElementById('album-name').textContent = track.album || 'Unknown Album'; + document.getElementById('artist-name').textContent = _stripSourceIdPrefix(track.artist) || 'Unknown Artist'; + document.getElementById('album-name').textContent = _stripSourceIdPrefix(track.album) || 'Unknown Album'; // Check if title needs scrolling (similar to GUI app) setTimeout(() => { @@ -107,10 +130,48 @@ function setTrackInfo(track) { document.getElementById('no-track-message').classList.add('hidden'); document.getElementById('media-player').classList.remove('idle'); + const gotoArtistBtn = document.getElementById('np-goto-artist'); + if (gotoArtistBtn) { + if (track.artist_id) { + gotoArtistBtn.href = buildArtistDetailPath(track.artist_id, track.artist_source || null); + gotoArtistBtn.style.pointerEvents = ''; + gotoArtistBtn.setAttribute('aria-disabled', 'false'); + gotoArtistBtn.tabIndex = 0; + } else { + gotoArtistBtn.href = '#'; + gotoArtistBtn.style.pointerEvents = 'none'; + gotoArtistBtn.setAttribute('aria-disabled', 'true'); + gotoArtistBtn.tabIndex = -1; + } + // Close the expanded now-playing modal when the user navigates + // to the artist page — otherwise the modal sits open over the + // page they just opened. ``_npGotoArtistHandlerAttached`` flag + // keeps us from binding multiple listeners across setTrackInfo + // calls (fires on every track change). + if (!gotoArtistBtn._npGotoArtistHandlerAttached) { + gotoArtistBtn.addEventListener('click', () => { + if (gotoArtistBtn.getAttribute('aria-disabled') === 'true') return; + try { closeNowPlayingModal(); } catch (e) { console.debug('closeNowPlayingModal failed:', e); } + }); + gotoArtistBtn._npGotoArtistHandlerAttached = true; + } + } + // Sync expanded player and media session updateNpTrackInfo(); updateMediaSessionMetadata(); updateMediaSessionPlaybackState(); + + // Kick off lyrics fetch for the new track. The panel stays + // collapsed by default — fetching in the background means the + // user gets instant lyrics the first time they expand it. + _npLyricsLoadForTrack({ + title: track.title, + artist: track.artist, + album: track.album, + is_library: track.is_library, + filename: track.filename, + }); } function checkAndEnableScrolling(element, text) { @@ -184,6 +245,14 @@ function clearTrack() { document.getElementById('no-track-message').classList.remove('hidden'); document.getElementById('media-player').classList.add('idle'); + const gotoArtistBtn = document.getElementById('np-goto-artist'); + if (gotoArtistBtn) { + gotoArtistBtn.href = '#'; + gotoArtistBtn.style.pointerEvents = 'none'; + gotoArtistBtn.setAttribute('aria-disabled', 'true'); + gotoArtistBtn.tabIndex = -1; + } + // Reset queue state npQueue = []; npQueueIndex = -1; @@ -899,6 +968,202 @@ function updateAudioProgress() { // Sync expanded player modal if (npModalOpen) updateNpProgress(); + + // Sync lyrics highlight when synced LRC is loaded. + if (_npLyricsState.synced && _npLyricsState.lines.length) { + _npLyricsHighlight(audioPlayer.currentTime); + } +} + +// ───────────────────────────────────────────────────────────────── +// Lyrics panel (now-playing modal) +// ───────────────────────────────────────────────────────────────── + +// Module-level state for the currently-loaded lyrics. Reset on each +// track change. ``lines`` is an array of {time, text} for synced +// lyrics or null for plain text. ``activeIndex`` tracks the last +// highlighted line to avoid re-rendering on every timeupdate tick. +const _npLyricsState = { + trackKey: null, + lines: [], + synced: false, + activeIndex: -1, + fetchInFlight: false, + autoOpen: false, +}; + +function _npLyricsResetUI() { + const content = document.getElementById('np-lyrics-content'); + const status = document.getElementById('np-lyrics-status'); + if (content) content.innerHTML = '
No lyrics loaded
'; + if (status) status.textContent = ''; +} + +function _npLyricsParseLrc(synced) { + // Parse a standard LRC string. Lines without a timestamp are + // dropped (metadata tags like ``[ti:Title]`` aren't lyrics). The + // same line can carry multiple timestamps — emit one entry per + // timestamp so seeks land correctly when a chorus repeats. + const out = []; + if (!synced) return out; + const re = /\[(\d+):(\d+(?:\.\d+)?)\]/g; + synced.split(/\r?\n/).forEach(raw => { + const stamps = []; + let m; + re.lastIndex = 0; + while ((m = re.exec(raw)) !== null) { + const minutes = parseInt(m[1], 10); + const seconds = parseFloat(m[2]); + if (!Number.isFinite(minutes) || !Number.isFinite(seconds)) continue; + stamps.push(minutes * 60 + seconds); + } + if (!stamps.length) return; + const text = raw.replace(re, '').trim(); + stamps.forEach(t => out.push({ time: t, text })); + }); + out.sort((a, b) => a.time - b.time); + return out; +} + +function _npLyricsRenderSynced(lines) { + const content = document.getElementById('np-lyrics-content'); + if (!content) return; + if (!lines.length) { + content.innerHTML = '
No timestamped lyrics for this track
'; + return; + } + content.innerHTML = lines.map((line, idx) => { + const safe = (line.text || '').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])) || ' '; + return `
${safe}
`; + }).join(''); +} + +function _npLyricsRenderPlain(text) { + const content = document.getElementById('np-lyrics-content'); + if (!content) return; + const safe = (text || '').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); + content.innerHTML = `
${safe.replace(/\n/g, '
')}
`; +} + +function _npLyricsHighlight(currentTime) { + const { lines } = _npLyricsState; + if (!lines.length) return; + let idx = -1; + // Binary-search style linear scan — N is small (200 lines max). + for (let i = 0; i < lines.length; i++) { + if (lines[i].time <= currentTime) idx = i; + else break; + } + if (idx === _npLyricsState.activeIndex) return; + _npLyricsState.activeIndex = idx; + const content = document.getElementById('np-lyrics-content'); + if (!content) return; + content.querySelectorAll('.np-lyrics-line').forEach((el, i) => { + el.classList.remove('active', 'passed', 'upcoming'); + if (i === idx) el.classList.add('active'); + else if (i < idx) el.classList.add('passed'); + else el.classList.add('upcoming'); + }); + const activeEl = content.querySelector('.np-lyrics-line.active'); + if (activeEl) { + // Smooth-scroll the active line into the middle of the lyrics body. + const body = document.getElementById('np-lyrics-body'); + if (body) { + const bodyRect = body.getBoundingClientRect(); + const lineRect = activeEl.getBoundingClientRect(); + const targetTop = (lineRect.top - bodyRect.top) - (bodyRect.height / 2) + (lineRect.height / 2); + body.scrollTo({ top: body.scrollTop + targetTop, behavior: 'smooth' }); + } + } +} + +function _npLyricsTrackKey(track) { + if (!track) return null; + return `${track.title || ''}|${track.artist || ''}|${track.album || ''}`; +} + +async function _npLyricsLoadForTrack(track) { + const key = _npLyricsTrackKey(track); + if (!key) return; + if (_npLyricsState.trackKey === key) return; // already loaded + if (_npLyricsState.fetchInFlight) return; + _npLyricsState.trackKey = key; + _npLyricsState.lines = []; + _npLyricsState.synced = false; + _npLyricsState.activeIndex = -1; + _npLyricsResetUI(); + const status = document.getElementById('np-lyrics-status'); + if (status) status.textContent = 'Fetching…'; + _npLyricsState.fetchInFlight = true; + try { + const resp = await fetch('/api/lyrics/fetch', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + title: _stripSourceIdPrefix(track.title) || '', + artist: _stripSourceIdPrefix(track.artist) || '', + album: _stripSourceIdPrefix(track.album) || '', + duration: Math.round(audioPlayer?.duration || 0), + file_path: track.is_library ? track.filename : null, + }), + }); + const data = await resp.json(); + if (_npLyricsState.trackKey !== key) return; // track changed mid-fetch + if (data && data.success) { + if (data.synced) { + const parsed = _npLyricsParseLrc(data.synced); + if (parsed.length) { + _npLyricsState.synced = true; + _npLyricsState.lines = parsed; + _npLyricsRenderSynced(parsed); + if (status) status.textContent = 'Synced'; + return; + } + } + if (data.plain) { + _npLyricsState.synced = false; + _npLyricsState.lines = []; + _npLyricsRenderPlain(data.plain); + if (status) status.textContent = 'Plain'; + return; + } + } + const content = document.getElementById('np-lyrics-content'); + if (content) content.innerHTML = '
No lyrics found
'; + if (status) status.textContent = ''; + } catch (e) { + console.debug('lyrics fetch failed:', e); + const content = document.getElementById('np-lyrics-content'); + if (content) content.innerHTML = '
Lyrics unavailable
'; + if (status) status.textContent = ''; + } finally { + _npLyricsState.fetchInFlight = false; + } +} + +function _npLyricsTogglePanel(forceOpen = null) { + const panel = document.getElementById('np-lyrics-panel'); + const body = document.getElementById('np-lyrics-body'); + const toggle = document.getElementById('np-lyrics-toggle'); + if (!panel || !body || !toggle) return; + const willOpen = forceOpen === null ? body.classList.contains('hidden') : forceOpen; + if (willOpen) { + body.classList.remove('hidden'); + panel.classList.remove('collapsed'); + toggle.setAttribute('aria-expanded', 'true'); + } else { + body.classList.add('hidden'); + panel.classList.add('collapsed'); + toggle.setAttribute('aria-expanded', 'false'); + } +} + +function _npLyricsInit() { + const toggle = document.getElementById('np-lyrics-toggle'); + if (toggle && !toggle._lyricsBound) { + toggle.addEventListener('click', () => _npLyricsTogglePanel()); + toggle._lyricsBound = true; + } } function onAudioEnded() { @@ -1231,15 +1496,11 @@ function initExpandedPlayer() { }); } - // Action button (Go to Artist) + // Action link (Go to Artist) const gotoArtistBtn = document.getElementById('np-goto-artist'); if (gotoArtistBtn) { - gotoArtistBtn.addEventListener('click', () => { - if (currentTrack && currentTrack.artist_id) { - closeNowPlayingModal(); - navigateToArtistDetail(currentTrack.artist_id, currentTrack.artist || ''); - } - }); + gotoArtistBtn.style.textDecoration = 'none'; + gotoArtistBtn.style.color = 'inherit'; } // Buffering state listeners on audioPlayer if (audioPlayer) { @@ -1268,6 +1529,10 @@ function openNowPlayingModal() { overlay.classList.remove('hidden'); document.body.style.overflow = 'hidden'; syncExpandedPlayerUI(); + // Bind lyrics toggle (idempotent — only attaches once). Lyrics + // fetch fires from setTrackInfo so by the time the modal opens + // the panel is usually already populated. + _npLyricsInit(); // Start visualizer if already playing if (isPlaying) { npInitVisualizer(); npStartVisualizerLoop(); } } @@ -2396,4 +2661,3 @@ function updateMediaSessionPlaybackState() { } // =============================== - diff --git a/webui/static/mobile.css b/webui/static/mobile.css index 8f292d98..043ea773 100644 --- a/webui/static/mobile.css +++ b/webui/static/mobile.css @@ -2750,323 +2750,6 @@ } } -/* ====================================== - STATS PAGE — Mobile - ====================================== */ - -@media (max-width: 768px) { - .stats-container { - padding: 12px !important; - margin: 8px !important; - gap: 14px; - border-radius: 16px; - } - - .stats-header { - flex-direction: column; - align-items: flex-start; - padding: 12px !important; - margin: -12px -12px 0 -12px !important; - gap: 10px; - border-top-left-radius: 16px; - border-top-right-radius: 16px; - } - - .stats-header-title { - gap: 8px; - } - - .stats-header-title .page-header-icon { - width: 24px; - height: 24px; - } - - .stats-header-title h1 { - font-size: 18px; - } - - .stats-header-controls { - width: 100%; - flex-direction: column; - align-items: flex-start !important; - gap: 8px; - } - - .stats-time-range { - display: flex; - flex-wrap: wrap; - gap: 6px; - width: 100%; - } - - .stats-range-btn { - padding: 6px 12px; - font-size: 11px; - flex: 1; - text-align: center; - } - - .stats-sync-controls { - width: 100%; - justify-content: space-between; - } - - .stats-last-synced { - font-size: 11px; - } - - .stats-sync-btn { - padding: 6px 12px; - font-size: 12px; - } - - /* Overview cards — 2 columns */ - .stats-overview { - grid-template-columns: repeat(2, 1fr) !important; - gap: 8px; - } - - .stats-card { - padding: 12px 10px; - border-radius: 10px; - } - - .stats-card-value { - font-size: 18px !important; - } - - .stats-card-label { - font-size: 9px; - letter-spacing: 0.04em; - } - - /* Main grid — single column */ - .stats-main-grid { - grid-template-columns: 1fr !important; - gap: 14px; - } - - .stats-left-col, - .stats-right-col { - width: 100% !important; - gap: 14px; - } - - .stats-section-card { - padding: 12px; - border-radius: 12px; - } - - .stats-section-title { - font-size: 13px; - margin-bottom: 10px; - } - - /* Timeline chart — shrink height */ - .stats-section-card > div[style*="height:220px"], - .stats-section-card > div[style*="height: 220px"] { - height: 160px !important; - } - - /* Genre chart — stack vertically, shrink canvas */ - .stats-genre-chart-container { - flex-direction: column !important; - align-items: center; - gap: 12px; - } - - .stats-genre-chart-container canvas { - width: 130px !important; - height: 130px !important; - } - - .stats-genre-legend { - width: 100%; - } - - .stats-genre-legend-item { - font-size: 11px; - } - - /* Top artists bubbles */ - .stats-artist-bubbles { - gap: 8px; - flex-wrap: wrap; - justify-content: center; - } - - .stats-artist-bubble { - max-width: 60px; - } - - .stats-bubble-img { - width: 44px !important; - height: 44px !important; - } - - .stats-bubble-name { - font-size: 9px; - } - - .stats-bubble-count { - font-size: 9px; - } - - .stats-bubble-bar-container { - height: 3px; - } - - /* Ranked lists */ - .stats-ranked-item { - padding: 8px 8px; - gap: 6px; - } - - .stats-ranked-num { - font-size: 11px; - min-width: 18px; - } - - .stats-ranked-img { - width: 32px; - height: 32px; - border-radius: 6px; - } - - .stats-ranked-info { - min-width: 0; - } - - .stats-ranked-name { - font-size: 12px; - } - - .stats-ranked-meta { - font-size: 10px; - } - - .stats-ranked-count { - font-size: 10px; - white-space: nowrap; - } - - .stats-play-btn { - width: 24px; - height: 24px; - font-size: 8px; - opacity: 1; - } - - .stats-play-btn-sm { - width: 20px; - height: 20px; - font-size: 7px; - opacity: 1; - } - - /* Recent plays */ - .stats-recent-item { - padding: 6px 8px; - gap: 6px; - } - - .stats-recent-title { - font-size: 12px; - } - - .stats-recent-artist { - font-size: 11px; - } - - .stats-recent-time { - font-size: 10px; - } - - /* Library health */ - .stats-full-width { - padding: 12px; - } - - .stats-health-grid { - grid-template-columns: 1fr !important; - gap: 10px; - } - - .stats-health-label { - font-size: 11px; - } - - .stats-health-value { - font-size: 16px; - } - - .stats-format-bar { - min-height: 20px; - border-radius: 6px; - } - - .stats-format-segment { - font-size: 9px; - } - - /* Enrichment coverage bars */ - .stats-enrichment { - gap: 6px; - } - - .stats-enrich-item { - gap: 6px; - } - - .stats-enrich-name { - font-size: 10px; - min-width: 55px; - } - - .stats-enrich-pct { - font-size: 10px; - min-width: 28px; - } - - /* DB storage chart */ - .stats-db-storage-wrap { - flex-direction: column; - align-items: center; - gap: 12px; - } - - .stats-db-chart-container { - width: 140px; - height: 140px; - } - - .stats-db-chart-container canvas { - width: 140px !important; - height: 140px !important; - } - - .stats-db-total-value { - font-size: 16px; - } - - .stats-db-legend { - width: 100%; - } - - /* Empty state */ - .stats-empty-icon { - font-size: 40px; - } - - .stats-empty h3 { - font-size: 16px; - } - - .stats-empty p { - font-size: 13px; - } -} - /* ====================================== ARTIST ENRICHMENT RINGS — Mobile ====================================== */ diff --git a/webui/static/pages-extra.js b/webui/static/pages-extra.js index 95ec45bb..47a65710 100644 --- a/webui/static/pages-extra.js +++ b/webui/static/pages-extra.js @@ -1821,6 +1821,11 @@ async function _serverSelectTrack(trackIndex, mode, newTrackId, el) { for (let k = 0; k < trackIndex; k++) { if (_serverEditorState.tracks[k]?.server_track) serverPos++; } + // source_track carries source_track_id (Spotify ID) when this + // came from a mirrored playlist — the backend uses it to + // persist the Find & Add selection as a permanent match + // override so future syncs auto-pair without user action. + const srcTrack = track.source_track || {}; response = await fetch(`/api/server/playlist/${_serverEditorState.playlistId}/add-track`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -1828,6 +1833,9 @@ async function _serverSelectTrack(trackIndex, mode, newTrackId, el) { track_id: newTrackId, playlist_name: _serverEditorState.playlistName, position: serverPos, + source_track_id: srcTrack.source_track_id || '', + source_title: srcTrack.name || '', + source_artist: srcTrack.artist || '', }) }); } @@ -2240,6 +2248,10 @@ function _updateDlNavBadge(count) { badge.classList.add('hidden'); } } + const dlBtn = document.querySelector('.nav-button[data-page="active-downloads"]'); + if (dlBtn) { + dlBtn.classList.toggle('nav-downloads-active', count > 0); + } } function _adlRender() { @@ -2433,6 +2445,74 @@ function _adlEsc(str) { return String(str).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); } +function _adlBundleProgressPercent(bundle) { + if (!bundle) return 0; + const raw = bundle.progress_percent ?? bundle.progress ?? 0; + let progress = Number(raw); + if (!Number.isFinite(progress)) progress = 0; + if (progress <= 1) progress *= 100; + return Math.max(0, Math.min(100, Math.round(progress))); +} + +function _adlFormatBytes(bytes) { + const value = Number(bytes); + if (!Number.isFinite(value) || value <= 0) return ''; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let size = value; + let unit = 0; + while (size >= 1024 && unit < units.length - 1) { + size /= 1024; + unit += 1; + } + const decimals = size >= 10 || unit === 0 ? 0 : 1; + return `${size.toFixed(decimals)} ${units[unit]}`; +} + +function _adlFormatSpeed(bytesPerSecond) { + const formatted = _adlFormatBytes(bytesPerSecond); + return formatted ? `${formatted}/s` : ''; +} + +function _adlSourceLabel(source) { + const labels = { + torrent: 'Torrent', + usenet: 'Usenet', + soulseek: 'Soulseek', + youtube: 'YouTube', + tidal: 'Tidal', + qobuz: 'Qobuz', + hifi: 'HiFi', + deezer_dl: 'Deezer', + amazon: 'Amazon', + lidarr: 'Lidarr', + soundcloud: 'SoundCloud' + }; + const key = String(source || '').toLowerCase(); + return labels[key] || (source ? String(source) : 'Release'); +} + +function _adlBundleStateLabel(state) { + const labels = { + searching: 'searching for release', + downloading: 'downloading release', + staged: 'matching tracks', + failed: 'release failed' + }; + const key = String(state || '').toLowerCase(); + return labels[key] || (state ? String(state).replace(/_/g, ' ') : 'downloading release'); +} + +function _adlBundleProgressText(bundle) { + const pct = _adlBundleProgressPercent(bundle); + const source = _adlSourceLabel(bundle && bundle.source); + const state = _adlBundleStateLabel(bundle && bundle.state); + const release = bundle && bundle.release ? ` - ${bundle.release}` : ''; + const speed = _adlFormatSpeed(bundle && bundle.speed); + const size = _adlFormatBytes(bundle && bundle.size); + const detail = speed || size ? ` (${[speed, size].filter(Boolean).join(' of ')})` : ''; + return `${source} ${state} ${pct}%${release}${detail}`; +} + async function adlClearCompleted() { try { const resp = await fetch('/api/downloads/clear-completed', { method: 'POST' }); @@ -2492,12 +2572,16 @@ function _adlRenderBatchPanel() { const colorStyle = colorIdx >= 0 ? `border-left-color: rgba(var(--batch-color-${colorIdx}), 0.6)` : ''; const isExpanded = _adlExpandedBatches.has(batch.batch_id); const isFiltered = _adlFilterBatchId === batch.batch_id; + const albumBundle = batch.album_bundle || null; + const bundleProgress = _adlBundleProgressPercent(albumBundle); const total = batch.total || 1; const done = batch.completed + batch.failed; - const pct = Math.round((done / total) * 100); + const pct = batch.phase === 'album_downloading' + ? bundleProgress + : Math.round((done / total) * 100); const hasFailed = batch.failed > 0; const isTerminal = batch.phase === 'complete' || batch.phase === 'cancelled' || batch.phase === 'error'; - const isActive = batch.phase === 'downloading' && batch.active > 0; + const isActive = (batch.phase === 'downloading' && batch.active > 0) || batch.phase === 'album_downloading'; // Fade progress for completing batches let fadeStyle = ''; @@ -2520,6 +2604,9 @@ function _adlRenderBatchPanel() { if (batch.phase === 'analysis') { phaseText = 'Analyzing...'; phaseIcon = ''; + } else if (batch.phase === 'album_downloading') { + phaseText = _adlBundleProgressText(albumBundle); + phaseIcon = ''; } else if (batch.phase === 'downloading') { phaseText = `${batch.completed}/${total} tracks`; if (batch.active > 0) phaseIcon = ''; @@ -2583,7 +2670,9 @@ function _adlRenderBatchPanel() {
`; }).join(''); } else { - tracksHtml = '
No tracks loaded
'; + tracksHtml = batch.phase === 'album_downloading' + ? '
Downloading one release first. Track matching starts after staging.
' + : '
No tracks loaded
'; } } diff --git a/webui/static/search.js b/webui/static/search.js index 8362e774..7a2214fc 100644 --- a/webui/static/search.js +++ b/webui/static/search.js @@ -317,11 +317,7 @@ function initializeSearchModeToggle() { name: artist.name, meta: 'In Your Library', badge: { text: 'Library', class: 'enh-badge-library' }, - onClick: () => { - console.log(`🎵 Opening library artist detail: ${artist.name} (ID: ${artist.id})`); - hideDropdown(); - navigateToArtistDetail(artist.id, artist.name); - } + href: buildArtistDetailPath(artist.id), }) ); @@ -337,12 +333,7 @@ function initializeSearchModeToggle() { name: artist.name, meta: 'Artist', badge: sourceBadge, - onClick: () => { - const sourceOverride = searchController.state.activeSource; - console.log(`🎵 Opening artist detail: ${artist.name} (ID: ${artist.id}, source: ${sourceOverride})`); - hideDropdown(); - navigateToArtistDetail(artist.id, artist.name, sourceOverride || null); - } + href: buildArtistDetailPath(artist.id, searchController.state.activeSource || null), }) ); @@ -1184,6 +1175,14 @@ async function loadInitialData() { return; } + if (targetPage === 'artist-detail') { + const artistRoute = typeof parseArtistDetailPath === 'function' ? parseArtistDetailPath() : null; + if (artistRoute && typeof navigateToArtistDetail === 'function') { + navigateToArtistDetail(artistRoute.artistId, '', artistRoute.source); + } + return; + } + // Always apply the target page to the legacy shell chrome. const router = getWebRouter(); const route = router?.routeManifest?.find((entry) => entry.pageId === targetPage); @@ -1197,7 +1196,7 @@ async function loadInitialData() { return; } - navigateToPage(targetPage, { skipRouteChange: true, forceReload: true }); + navigateToPage(targetPage, { forceReload: true }); } catch (error) { console.error('Error loading initial data:', error); } diff --git a/webui/static/settings.js b/webui/static/settings.js index 5364f643..7c3a127d 100644 --- a/webui/static/settings.js +++ b/webui/static/settings.js @@ -71,8 +71,7 @@ function _isMetadataSourceSelectable(source) { function _metadataSourceFallback(source) { if (source === 'spotify') return 'deezer'; - if (source === 'discogs') return 'itunes'; - return 'itunes'; + return 'deezer'; } function focusServiceSettingsSection(service, message) { @@ -100,7 +99,7 @@ function sanitizeMetadataSourceSelection({ quiet = true } = {}) { const select = document.getElementById('metadata-fallback-source'); if (!select) return false; - const selectedSource = select.value || 'itunes'; + const selectedSource = select.value || 'deezer'; if (_isMetadataSourceSelectable(selectedSource)) { select.dataset.lastValidSource = selectedSource; return false; @@ -595,12 +594,16 @@ const HYBRID_SOURCES = [ { id: 'qobuz', name: 'Qobuz', icon: 'https://www.svgrepo.com/show/504778/qobuz.svg', emoji: '🎧' }, { id: 'hifi', name: 'HiFi', icon: null, emoji: '🎶' }, { id: 'deezer_dl', name: 'Deezer', icon: 'https://www.svgrepo.com/show/519734/deezer.svg', emoji: '🎧' }, + { id: 'amazon', name: 'Amazon Music', icon: null, emoji: '🛒' }, { id: 'lidarr', name: 'Lidarr', icon: null, emoji: '📦' }, { id: 'soundcloud', name: 'SoundCloud', icon: 'https://www.svgrepo.com/show/452219/soundcloud.svg', emoji: '☁️' }, + { id: 'torrent', name: 'Torrent', icon: null, emoji: '🧲' }, + { id: 'usenet', name: 'Usenet', icon: null, emoji: '📰' }, ]; +const ALBUM_LEVEL_HYBRID_SOURCES = new Set(['soulseek', 'torrent', 'usenet']); let _hybridSourceOrder = ['soulseek', 'youtube']; -let _hybridSourceEnabled = { soulseek: true, youtube: true, tidal: false, qobuz: false, hifi: false, deezer_dl: false, lidarr: false, soundcloud: false }; +let _hybridSourceEnabled = { soulseek: true, youtube: true, tidal: false, qobuz: false, hifi: false, deezer_dl: false, amazon: false, lidarr: false, soundcloud: false, torrent: false, usenet: false }; let _hybridVisualOrder = null; // Full visual order including disabled sources function buildHybridSourceList() { @@ -623,6 +626,13 @@ function buildHybridSourceList() { const enabled = _hybridSourceEnabled[srcId] !== false; const isInOrder = _hybridSourceOrder.includes(srcId); const priorityNum = isInOrder && enabled ? _hybridSourceOrder.indexOf(srcId) + 1 : ''; + const canOwnAlbum = enabled && priorityNum === 1 && ALBUM_LEVEL_HYBRID_SOURCES.has(srcId); + const sourceLevel = canOwnAlbum ? 'Album-level' : 'Track-level'; + const sourceLevelClass = canOwnAlbum ? 'album' : 'track'; + const sourceLevelTitle = canOwnAlbum + ? 'This first source can download a whole album release before per-track fallback.' + : 'This source runs as per-track fallback in the current hybrid order.'; + const sourceLevelBadge = `${sourceLevel}`; const item = document.createElement('div'); item.className = `hybrid-source-item${enabled ? '' : ' disabled'}`; @@ -639,6 +649,7 @@ function buildHybridSourceList() { : `${src.emoji}` } ${src.name} + ${sourceLevelBadge} ${priorityNum}