diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5e87934d..0527e3a6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,9 +9,9 @@ on: workflow_dispatch: inputs: version_tag: - description: 'Version tag (e.g. 2.5.3)' + description: 'Version tag (e.g. 2.5.4)' required: true - default: '2.5.3' + default: '2.5.4' jobs: build-and-push: diff --git a/core/acoustid_verification.py b/core/acoustid_verification.py index bd2e917c..d1b87918 100644 --- a/core/acoustid_verification.py +++ b/core/acoustid_verification.py @@ -16,6 +16,7 @@ from enum import Enum from utils.logging_config import get_logger from core.acoustid_client import AcoustIDClient from core.matching_engine import MusicMatchingEngine +from core.matching.version_mismatch import is_acceptable_version_mismatch from core.musicbrainz_client import MusicBrainzClient logger = get_logger("acoustid.verification") @@ -485,12 +486,33 @@ class AcoustIDVerification: expected_version = _detect_title_version(expected_track_name) matched_version = _detect_title_version(matched_title) if expected_version != matched_version: - msg = ( - f"Version mismatch: expected '{expected_track_name}' ({expected_version}) " - f"but file is '{matched_title}' ({matched_version})" - ) - logger.warning(f"AcoustID verification FAILED (version mismatch) - {msg}") - return VerificationResult.FAIL, msg + # Issue #607 (AfonsoG6): MusicBrainz often stores live + # recordings with bare titles ("Clarity") while the + # release entry carries the venue annotation ("Clarity + # (Live at Blossom Music Center, ...)"). The fingerprint + # correctly identifies the LIVE recording; only the + # title text is bare. Helper accepts the one-sided bare + # case when fingerprint + bare-title + artist all agree. + # Two-sided version mismatches (live vs remix etc) stay + # strict — those are genuinely different recordings. + if is_acceptable_version_mismatch( + expected_version, matched_version, + fingerprint_score=best_score, + title_similarity=title_sim, + artist_similarity=artist_sim, + ): + logger.info( + f"AcoustID version annotation differs (expected={expected_version}, " + f"matched={matched_version}) but fingerprint+title+artist all match — " + f"accepting (likely MB metadata gap on a live/version-annotated recording)" + ) + else: + msg = ( + f"Version mismatch: expected '{expected_track_name}' ({expected_version}) " + f"but file is '{matched_title}' ({matched_version})" + ) + logger.warning(f"AcoustID verification FAILED (version mismatch) - {msg}") + return VerificationResult.FAIL, msg # Step 5: Decide pass/fail based on similarity if title_sim >= TITLE_MATCH_THRESHOLD and artist_sim >= ARTIST_MATCH_THRESHOLD: @@ -593,12 +615,19 @@ class AcoustIDVerification: # downloading Mr. Morale: three tracks (Rich Interlude, Savior # Interlude, Savior) all received the wrong R.O.T.C audio file # because of this leak. - top = recordings[0] - top_title = top.get('title', '?') or '' - top_artist = top.get('artist', '?') or '' + # Use the BEST matching recording's strings here (not + # `recordings[0]`) so the failure message reports the same + # candidate the title/artist similarity scores came from. + # Issue #607 (AfonsoG6) example 1: the prior code mixed + # `recordings[0]`'s strings (which can be empty) with + # `best_rec`'s scores, producing nonsense reasons like + # "file identified as '' by '' (artist=100%)" when a later + # recording in the list scored well on artist. + display_title = matched_title or '?' + display_artist = matched_artist or '?' has_non_ascii = ( any(ord(c) > 127 for c in (expected_track_name or '')) - or any(ord(c) > 127 for c in top_title) + or any(ord(c) > 127 for c in display_title) ) language_script_skip = ( best_score >= 0.95 @@ -618,18 +647,16 @@ class AcoustIDVerification: ) msg = ( f"Title/artist mismatch but fingerprint confidence very high ({best_score:.2f}): " - f"AcoustID='{top_title}' by '{top_artist}', " + f"AcoustID='{display_title}' by '{display_artist}', " f"expected '{expected_track_name}' by '{expected_artist_name}' — " f"{reason}" ) logger.info(f"AcoustID verification SKIPPED (high confidence) - {msg}") return VerificationResult.SKIP, msg - # Low fingerprint score + no metadata match — file is likely wrong - # `top`, `top_title`, `top_artist` already resolved above for the - # skip-eligibility check. + # Low fingerprint score + no metadata match — file is likely wrong. msg = ( - f"Audio mismatch: file identified as '{top_title}' by '{top_artist}', " + f"Audio mismatch: file identified as '{display_title}' by '{display_artist}', " f"expected '{expected_track_name}' by '{expected_artist_name}' " f"(title={title_sim:.0%}, artist={artist_sim:.0%})" ) diff --git a/core/amazon_client.py b/core/amazon_client.py new file mode 100644 index 00000000..8d85e3cd --- /dev/null +++ b/core/amazon_client.py @@ -0,0 +1,773 @@ +"""Amazon Music metadata client backed by a T2Tunes proxy instance. + +T2Tunes exposes the Amazon Music catalog (search, album metadata, stream info) +through a simple REST API. This module wraps those endpoints and normalises the +responses into the same Track / Artist / Album dataclass shape used by +DeezerClient and iTunesClient. + +Endpoints used: + GET /api/status + GET /api/amazon-music/search + GET /api/amazon-music/metadata + GET /api/amazon-music/media-from-asin + +Config keys (all optional — fall back to public defaults): + amazon.base_url T2Tunes instance URL (default: https://t2tunes.site) + amazon.country ISO-3166 country code (default: US) + amazon.preferred_codec Preferred audio codec (default: flac) +""" + +from __future__ import annotations + +import re +import threading +import time +from dataclasses import dataclass +from typing import Any, Dict, Iterator, List, Optional +from urllib.parse import urljoin + +import requests + +from config.settings import config_manager +from core.api_call_tracker import api_call_tracker +from utils.logging_config import get_logger + +logger = get_logger("amazon_client") + +# Strips featuring credits like "Artist feat. X", "Artist ft. Y" so artist +# deduplication works on the primary artist name only. +_FEAT_RE = re.compile(r'\s+(?:feat(?:uring)?\.?|ft\.?)\s+.*', re.IGNORECASE) + +# Strips the Explicit marker — explicit is treated as the default version. +# Clean/Edited/Censored stay in the name so users can distinguish them. +_EDITION_RE = re.compile(r'\s*[\[\(]explicit[\]\)]', re.IGNORECASE) + + +def _primary_artist(name: str) -> str: + return _FEAT_RE.sub('', name).strip() + + +def _strip_edition(name: str) -> str: + return _EDITION_RE.sub('', name).strip() + + +def _unslugify(name: str) -> str: + """Convert a slug-form artist ID (e.g. 'kendrick_lamar') to a search name.""" + return name.replace('_', ' ') + + +DEFAULT_BASE_URL = "https://t2tunes.site" +DEFAULT_COUNTRY = "US" +DEFAULT_CODEC = "flac" +MIN_API_INTERVAL = 0.5 # seconds — T2Tunes has no published rate limit + +_last_api_call: float = 0.0 +_api_call_lock = threading.Lock() + +_META_CACHE_TTL = 300 # seconds +_meta_cache: Dict[str, tuple] = {} # asin -> (fetched_at, meta_dict) +_meta_cache_lock = threading.Lock() + + +class AmazonClientError(RuntimeError): + """Raised on unrecoverable T2Tunes API errors.""" + + +# --------------------------------------------------------------------------- +# Dataclasses — field layout matches DeezerClient / iTunesClient exactly +# --------------------------------------------------------------------------- + +@dataclass +class Track: + id: str # Amazon track ASIN + name: str + artists: List[str] + album: str + duration_ms: int + popularity: int + preview_url: Optional[str] = None + external_urls: Optional[Dict[str, str]] = None + image_url: Optional[str] = None + release_date: Optional[str] = None + track_number: Optional[int] = None + disc_number: Optional[int] = None + album_type: Optional[str] = None + total_tracks: Optional[int] = None + isrc: Optional[str] = None + + @classmethod + def from_search_hit(cls, doc: Dict[str, Any]) -> "Track": + return cls( + id=str(doc.get("asin") or ""), + name=str(doc.get("title") or ""), + artists=[str(doc.get("artistName") or "Unknown Artist")], + album=str(doc.get("albumName") or ""), + duration_ms=int(doc.get("duration") or 0) * 1000, + popularity=0, + isrc=str(doc.get("isrc") or "") or None, + ) + + @classmethod + def from_stream_info( + cls, + stream: "T2TunesStreamInfo", + album_meta: Optional[Dict[str, Any]] = None, + ) -> "Track": + album = album_meta or {} + return cls( + id=stream.asin, + name=stream.title, + artists=[stream.artist] if stream.artist else ["Unknown Artist"], + album=stream.album, + duration_ms=0, + popularity=0, + image_url=album.get("image"), + release_date=album.get("release_date"), + total_tracks=album.get("trackCount"), + isrc=stream.isrc or None, + ) + + +@dataclass +class Artist: + id: str # Slugified artist name — T2Tunes exposes no artist IDs + name: str + popularity: int + genres: List[str] + followers: int + image_url: Optional[str] = None + external_urls: Optional[Dict[str, str]] = None + + @classmethod + def from_name(cls, name: str) -> "Artist": + slug = name.lower().replace(" ", "_") + return cls(id=slug, name=name, popularity=0, genres=[], followers=0) + + +@dataclass +class Album: + id: str # Amazon album ASIN + name: str + artists: List[str] + release_date: str + total_tracks: int + album_type: str + image_url: Optional[str] = None + external_urls: Optional[Dict[str, str]] = None + explicit: Optional[bool] = None + + @classmethod + def from_search_hit(cls, doc: Dict[str, Any]) -> "Album": + return cls( + id=str(doc.get("albumAsin") or doc.get("asin") or ""), + name=str(doc.get("albumName") or doc.get("title") or ""), + artists=[str(doc.get("artistName") or "Unknown Artist")], + release_date="", + total_tracks=0, + album_type="album", + ) + + @classmethod + def from_metadata(cls, album_meta: Dict[str, Any], asin: str = "") -> "Album": + return cls( + id=str(album_meta.get("asin") or asin or ""), + name=str(album_meta.get("title") or ""), + artists=[str(album_meta.get("artistName") or "Unknown Artist")], + release_date=str(album_meta.get("release_date") or album_meta.get("releaseDate") or ""), + total_tracks=int(album_meta.get("trackCount") or 0), + album_type="album", + image_url=album_meta.get("image"), + explicit=album_meta.get("explicit"), + ) + + +# --------------------------------------------------------------------------- +# Internal dataclasses for raw T2Tunes response parsing +# --------------------------------------------------------------------------- + +@dataclass(frozen=True) +class T2TunesSearchItem: + asin: str + title: str + artist_name: str + item_type: str + album_name: str = "" + album_asin: str = "" + duration_seconds: int = 0 + isrc: str = "" + + @property + def is_album(self) -> bool: + return "album" in self.item_type.lower() + + @property + def is_track(self) -> bool: + return "track" in self.item_type.lower() + + +@dataclass(frozen=True) +class T2TunesStreamInfo: + asin: str + streamable: bool + codec: str + format: str + sample_rate: Optional[int] + stream_url: str + decryption_key: Optional[str] # hex-encoded AES key; None when stream is clear + title: str = "" + artist: str = "" + album: str = "" + isrc: str = "" + cover_url: str = "" + track_number: Optional[int] = None + disc_number: Optional[int] = None + genre: str = "" + label: str = "" + date: str = "" + + @property + def has_decryption_key(self) -> bool: + return bool(self.decryption_key) + + +# --------------------------------------------------------------------------- +# Rate-limit enforcement +# --------------------------------------------------------------------------- + +def _rate_limit() -> None: + global _last_api_call + with _api_call_lock: + elapsed = time.monotonic() - _last_api_call + if elapsed < MIN_API_INTERVAL: + time.sleep(MIN_API_INTERVAL - elapsed) + _last_api_call = time.monotonic() + api_call_tracker.record_call("amazon") + + +# --------------------------------------------------------------------------- +# Main client +# --------------------------------------------------------------------------- + +class AmazonClient: + """T2Tunes-backed Amazon Music metadata and stream-info client.""" + + def __init__( + self, + base_url: Optional[str] = None, + country: Optional[str] = None, + preferred_codec: Optional[str] = None, + timeout: int = 30, + session: Optional[Any] = None, + ) -> None: + self.base_url = (base_url or config_manager.get("amazon.base_url", DEFAULT_BASE_URL)).rstrip("/") + self.country = (country or config_manager.get("amazon.country", DEFAULT_COUNTRY)).upper() + self.preferred_codec = ( + preferred_codec or config_manager.get("amazon.preferred_codec", DEFAULT_CODEC) + ).lower() + self.timeout = timeout + self.session: Any = session or requests.Session() + if isinstance(self.session, requests.Session): + self.session.headers.update({ + "Accept": "application/json", + "User-Agent": "SoulSync/1.0", + "Referer": self.base_url, + }) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def reload_config(self) -> None: + self.base_url = config_manager.get("amazon.base_url", DEFAULT_BASE_URL).rstrip("/") + self.country = config_manager.get("amazon.country", DEFAULT_COUNTRY).upper() + self.preferred_codec = config_manager.get("amazon.preferred_codec", DEFAULT_CODEC).lower() + + def is_authenticated(self) -> bool: + """Return True when the T2Tunes instance reports Amazon Music as up.""" + try: + return str(self.status().get("amazonMusic", "")).lower() == "up" + except AmazonClientError: + return False + + # ------------------------------------------------------------------ + # Low-level API wrappers + # ------------------------------------------------------------------ + + def status(self) -> Dict[str, Any]: + return self._get_json("/api/status") + + def search_raw(self, query: str, *, types: str = "track,album") -> List[T2TunesSearchItem]: + data = self._get_json( + "/api/amazon-music/search", + params={"query": query, "types": types, "country": self.country}, + ) + return list(self._iter_search_items(data)) + + def album_metadata(self, asin: str) -> Dict[str, Any]: + return self._get_json( + "/api/amazon-music/metadata", + params={"asin": asin, "country": self.country}, + ) + + def media_from_asin(self, asin: str, codec: Optional[str] = None) -> List[T2TunesStreamInfo]: + effective_codec = (codec or self.preferred_codec).lower() + data = self._get_json( + "/api/amazon-music/media-from-asin", + params={"asin": asin, "country": self.country, "codec": effective_codec}, + ) + if isinstance(data, list): + return [self._parse_stream_info(item) for item in data if isinstance(item, dict)] + if isinstance(data, dict): + return [self._parse_stream_info(data)] + raise AmazonClientError(f"Unexpected media-from-asin response type: {type(data).__name__}") + + # ------------------------------------------------------------------ + # Metadata interface — mirrors DeezerClient / iTunesClient signatures + # ------------------------------------------------------------------ + + def search_tracks(self, query: str, limit: int = 20) -> List[Track]: + _rate_limit() + items = self.search_raw(query, types="track,album") + track_pairs: List[tuple] = [] # (Track, album_asin) + seen_album_asins: List[str] = [] + for item in items: + if not item.is_track: + continue + track = Track.from_search_hit({ + "asin": item.asin, + "title": _strip_edition(item.title), + "artistName": _primary_artist(item.artist_name), + "albumName": _strip_edition(item.album_name), + "albumAsin": item.album_asin, + "duration": item.duration_seconds, + "isrc": item.isrc, + }) + track_pairs.append((track, item.album_asin)) + if item.album_asin and item.album_asin not in seen_album_asins: + seen_album_asins.append(item.album_asin) + if len(track_pairs) >= limit: + break + album_metas = self._fetch_album_metas(seen_album_asins[:5]) + tracks: List[Track] = [] + for track, album_asin in track_pairs: + if album_asin and album_asin in album_metas: + meta = album_metas[album_asin] + track.image_url = meta.get("image") + track.release_date = str(meta.get("release_date") or meta.get("releaseDate") or "") + track.total_tracks = meta.get("trackCount") + tracks.append(track) + return tracks + + def search_artists(self, query: str, limit: int = 20) -> List[Artist]: + _rate_limit() + items = self.search_raw(query, types="track,album") + seen: Dict[str, Artist] = {} + artist_album_asin: Dict[str, str] = {} # artist name → first album ASIN seen + for item in items: + name = _primary_artist(item.artist_name) + if not name: + continue + if name not in seen: + seen[name] = Artist.from_name(name) + if name not in artist_album_asin and item.album_asin: + artist_album_asin[name] = item.album_asin + if len(seen) >= limit: + break + # T2Tunes has no artist images — use an album cover as stand-in. + unique_asins = list({v for v in artist_album_asin.values()})[:5] + album_metas = self._fetch_album_metas(unique_asins) + for name, artist in seen.items(): + asin = artist_album_asin.get(name) + if asin and asin in album_metas: + artist.image_url = album_metas[asin].get("image") + return list(seen.values()) + + def search_albums(self, query: str, limit: int = 20) -> List[Album]: + _rate_limit() + items = self.search_raw(query, types="track,album") + album_candidates: List[tuple] = [] # (Album, asin) + seen_keys: set = set() + for item in items: + if not item.is_album: + continue + album_asin = item.album_asin or item.asin + raw_name = item.album_name or item.title + display_name = _strip_edition(raw_name) + artist = _primary_artist(item.artist_name) + # Collapse Explicit/Clean variants: same normalised name + artist = same album + dedup_key = (display_name.lower(), artist.lower()) + if dedup_key in seen_keys: + continue + seen_keys.add(dedup_key) + album = Album.from_search_hit({ + "albumAsin": album_asin, + "albumName": display_name, + "artistName": artist, + }) + album_candidates.append((album, album_asin)) + if len(album_candidates) >= limit: + break + album_metas = self._fetch_album_metas([a for _, a in album_candidates[:10]]) + albums: List[Album] = [] + for album, asin in album_candidates: + if asin in album_metas: + meta = album_metas[asin] + album.image_url = meta.get("image") + album.release_date = str(meta.get("release_date") or meta.get("releaseDate") or "") + album.total_tracks = int(meta.get("trackCount") or 0) + albums.append(album) + return albums + + def get_track_details(self, asin: str) -> Optional[Dict[str, Any]]: + """Return a Spotify-compatible dict for a single track ASIN.""" + _rate_limit() + try: + streams = self.media_from_asin(asin) + except AmazonClientError: + return None + if not streams: + return None + s = streams[0] + + album_data: Dict[str, Any] = {} + try: + meta = self.album_metadata(asin) + albums = meta.get("albumList") + if isinstance(albums, list) and albums and isinstance(albums[0], dict): + album_data = albums[0] + except AmazonClientError: + pass + + return { + "id": s.asin, + "name": _strip_edition(s.title), + "artists": [{"name": _primary_artist(s.artist), "id": ""}], + "album": { + "id": album_data.get("asin", ""), + "name": _strip_edition(s.album), + "images": [{"url": album_data["image"]}] if album_data.get("image") else [], + "release_date": album_data.get("release_date") or album_data.get("releaseDate") or s.date or "", + "total_tracks": album_data.get("trackCount", 0), + }, + "duration_ms": 0, + "popularity": 0, + "external_urls": {"amazon": f"https://music.amazon.com/albums/{asin}"}, + "track_number": s.track_number, + "disc_number": s.disc_number, + "isrc": s.isrc, + "is_album_track": True, + "raw_data": { + "codec": s.codec, + "format": s.format, + "sample_rate": s.sample_rate, + "streamable": s.streamable, + "has_decryption_key": s.has_decryption_key, + }, + } + + def get_album(self, asin: str, include_tracks: bool = True) -> Optional[Dict[str, Any]]: + """Return a Spotify-compatible album dict.""" + _rate_limit() + try: + meta = self.album_metadata(asin) + except AmazonClientError: + return None + + albums = meta.get("albumList") + if not isinstance(albums, list) or not albums: + return None + album = albums[0] if isinstance(albums[0], dict) else {} + + result: Dict[str, Any] = { + "id": asin, + "name": _strip_edition(album.get("title", "")), + "artists": [{"name": _primary_artist(album.get("artistName", "")), "id": ""}], + "release_date": album.get("release_date") or album.get("releaseDate") or "", + "total_tracks": album.get("trackCount", 0), + "album_type": "album", + "images": [{"url": album["image"]}] if album.get("image") else [], + "external_urls": {"amazon": f"https://music.amazon.com/albums/{asin}"}, + "label": album.get("label", ""), + } + if include_tracks: + tracks_data = self.get_album_tracks(asin) or { + "items": [], "total": 0, "limit": 50, "next": None, + } + result["tracks"] = tracks_data + # Backfill release_date from stream tags when album metadata lacks it. + if not result["release_date"]: + items = tracks_data.get("items") or [] + for item in items: + rd = item.get("release_date") or "" + if rd and len(rd) >= 4: + result["release_date"] = rd + break + return result + + def get_album_tracks(self, asin: str) -> Optional[Dict[str, Any]]: + """Return album tracks in Spotify pagination format.""" + _rate_limit() + try: + streams = self.media_from_asin(asin) + except AmazonClientError: + return None + + # media_from_asin has no duration — enrich from search results which do. + duration_map: Dict[str, int] = {} # track asin → duration_ms + if streams: + album_name = _strip_edition(streams[0].album) + artist_name = _primary_artist(streams[0].artist) + try: + search_items = self.search_raw( + f"{album_name} {artist_name}", types="track,album" + ) + for item in search_items: + if item.album_asin == asin and item.duration_seconds: + duration_map[item.asin] = item.duration_seconds * 1000 + except Exception as exc: + logger.debug("Duration backfill failed for ASIN %s: %s", asin, exc) + + items = [ + { + "id": s.asin, + "name": _strip_edition(s.title), + "artists": [{"name": _primary_artist(s.artist), "id": ""}], + "duration_ms": duration_map.get(s.asin, 0), + "track_number": s.track_number, + "disc_number": s.disc_number, + "release_date": s.date or "", + "isrc": s.isrc, + } + for s in streams + ] + return {"items": items, "total": len(items), "limit": 50, "next": None} + + def get_artist(self, artist_name: str) -> Optional[Dict[str, Any]]: + """Return a Spotify-compatible artist dict inferred from search results.""" + _rate_limit() + search_name = _unslugify(artist_name) + try: + items = self.search_raw(search_name, types="track,album") + except AmazonClientError: + return None + name_lower = search_name.lower() + match = next( + (i for i in items if _primary_artist(i.artist_name).lower() == name_lower), + next((i for i in items if name_lower in _primary_artist(i.artist_name).lower()), None), + ) + if not match: + return None + return { + "id": match.artist_name.lower().replace(" ", "_"), + "name": match.artist_name, + "genres": [], + "popularity": 0, + "followers": {"total": 0}, + "images": [], + "external_urls": {}, + } + + def get_artist_albums( + self, + artist_name: str, + album_type: str = "album,single", + limit: int = 200, + ) -> List[Album]: + """Return albums for an artist inferred from search results.""" + _rate_limit() + search_name = _unslugify(artist_name) + try: + items = self.search_raw(f"{search_name} album", types="track,album") + except AmazonClientError: + return [] + album_candidates: List[tuple] = [] # (Album, asin) + seen_asins: set = set() + name_lower = search_name.lower() + for item in items: + if not item.is_album: + continue + if _primary_artist(item.artist_name).lower() != name_lower: + continue + album_asin = item.album_asin or item.asin + if album_asin in seen_asins: + continue + seen_asins.add(album_asin) + album = Album.from_search_hit({ + "albumAsin": album_asin, + "albumName": _strip_edition(item.album_name or item.title), + "artistName": _primary_artist(item.artist_name), + }) + album_candidates.append((album, album_asin)) + if len(album_candidates) >= limit: + break + + # Fetch metadata for art, release_date, track_count, and type inference. + # No explicit cap here — search_raw naturally bounds results to ~20 items, + # so album_candidates is already small. + asins_to_fetch = [a for _, a in album_candidates] + metas = self._fetch_album_metas(asins_to_fetch) + + albums: List[Album] = [] + for album, asin in album_candidates: + meta = metas.get(asin, {}) + if meta: + album.image_url = meta.get("image") + album.release_date = str(meta.get("release_date") or meta.get("releaseDate") or "") + total = int(meta.get("trackCount") or 0) + album.total_tracks = total + # T2Tunes doesn't expose release type — infer from track count. + # 1-track releases are singles; keep default "album" otherwise. + if total == 1: + album.album_type = "single" + albums.append(album) + return albums + + def get_track_features(self, track_id: str) -> Optional[Dict[str, Any]]: + """Not available from Amazon Music — returns None for compatibility.""" + return None + + def _get_artist_image_from_albums(self, artist_id: str) -> Optional[str]: + """Return an album cover as artist image stand-in (T2Tunes has no artist images).""" + search_name = _unslugify(artist_id) + try: + items = self.search_raw(search_name, types="track,album") + except AmazonClientError: + return None + name_lower = search_name.lower() + for item in items: + if _primary_artist(item.artist_name).lower() != name_lower: + continue + asin = item.album_asin or item.asin + if not asin: + continue + metas = self._fetch_album_metas([asin]) + if asin in metas and metas[asin].get("image"): + return metas[asin]["image"] + return None + + # ==================== Interface Aliases (match DeezerClient method names) ==================== + get_album_metadata = get_album + get_artist_info = get_artist + get_artist_albums_list = get_artist_albums + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _fetch_album_metas(self, asins: List[str]) -> Dict[str, Dict[str, Any]]: + """Parallel-fetch album metadata for up to N ASINs. Returns {asin: albumList[0]}.""" + if not asins: + return {} + metas: Dict[str, Dict[str, Any]] = {} + + def _fetch(asin: str) -> None: + now = time.monotonic() + with _meta_cache_lock: + entry = _meta_cache.get(asin) + if entry and (now - entry[0]) < _META_CACHE_TTL: + metas[asin] = entry[1] + return + _rate_limit() + try: + raw = self.album_metadata(asin) + lst = raw.get("albumList") + if isinstance(lst, list) and lst and isinstance(lst[0], dict): + meta = lst[0] + with _meta_cache_lock: + _meta_cache[asin] = (time.monotonic(), meta) + metas[asin] = meta + except Exception as exc: + logger.debug("Album metadata fetch failed for ASIN %s: %s", asin, exc) + + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=min(len(asins), 5)) as pool: + list(pool.map(_fetch, asins)) + return metas + + def _get_json(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any: + url = urljoin(f"{self.base_url}/", path.lstrip("/")) + try: + resp = self.session.get(url, params=params, timeout=self.timeout) + resp.raise_for_status() + except requests.HTTPError as exc: + raise AmazonClientError( + f"HTTP {exc.response.status_code} for {url}" + ) from exc + except requests.RequestException as exc: + raise AmazonClientError(f"Request failed for {url}: {exc}") from exc + try: + return resp.json() + except ValueError as exc: + preview = resp.text[:200].replace("\n", " ") + raise AmazonClientError(f"Response not JSON for {url}: {preview!r}") from exc + + @staticmethod + def _iter_search_items(response: Any) -> Iterator[T2TunesSearchItem]: + if not isinstance(response, dict): + raise AmazonClientError( + f"Unexpected search response type: {type(response).__name__}" + ) + for result in response.get("results") or []: + if not isinstance(result, dict): + continue + for hit in result.get("hits") or []: + if not isinstance(hit, dict): + continue + doc = hit.get("document") + if not isinstance(doc, dict): + continue + asin = str(doc.get("asin") or "") + if not asin: + continue + yield T2TunesSearchItem( + asin=asin, + title=str(doc.get("title") or ""), + artist_name=str(doc.get("artistName") or ""), + item_type=str(doc.get("__type") or ""), + album_name=str(doc.get("albumName") or ""), + album_asin=str(doc.get("albumAsin") or ""), + duration_seconds=int(doc.get("duration") or 0), + isrc=str(doc.get("isrc") or ""), + ) + + @staticmethod + def _parse_stream_info(item: Dict[str, Any]) -> T2TunesStreamInfo: + stream_info = item.get("streamInfo") if isinstance(item.get("streamInfo"), dict) else {} + tags = item.get("tags") if isinstance(item.get("tags"), dict) else {} + # T2Tunes API has a typo: "stremeable" in some responses + streamable = item.get("streamable") + if streamable is None: + streamable = item.get("stremeable") + raw_key = item.get("decryptionKey") + decryption_key = str(raw_key) if raw_key else None + + def _int_tag(key: str) -> Optional[int]: + v = tags.get(key) + try: + return int(v) if v is not None else None + except (TypeError, ValueError): + return None + + return T2TunesStreamInfo( + asin=str(item.get("asin") or ""), + streamable=bool(streamable), + codec=str(stream_info.get("codec") or ""), + format=str(stream_info.get("format") or ""), + sample_rate=( + stream_info.get("sampleRate") + if isinstance(stream_info.get("sampleRate"), int) + else None + ), + stream_url=str(stream_info.get("streamUrl") or ""), + decryption_key=decryption_key, + title=str(tags.get("title") or ""), + artist=str(tags.get("artist") or ""), + album=str(tags.get("album") or ""), + isrc=str(tags.get("isrc") or ""), + cover_url=str(item.get("coverUrl") or ""), + track_number=_int_tag("trackNumber"), + disc_number=_int_tag("discNumber"), + genre=str(tags.get("genre") or ""), + label=str(tags.get("label") or ""), + date=str(tags.get("date") or ""), + ) diff --git a/core/amazon_download_client.py b/core/amazon_download_client.py new file mode 100644 index 00000000..18e4bad1 --- /dev/null +++ b/core/amazon_download_client.py @@ -0,0 +1,452 @@ +"""Amazon Music download source plugin backed by a T2Tunes proxy. + +NOT yet wired into the app registry — validated in isolation only. +See tests/tools/test_amazon_download_client.py. + +Filename encoding (the DownloadSourcePlugin dispatch contract): + "{asin}||{display_name}" + e.g. "B09XYZ1234||Kendrick Lamar - Not Like Us" + +Codec preference order: FLAC → Opus → EAC3. + +Download flow (from Tubifarry reference implementation): + 1. GET stream_url → encrypted bytes on disk + 2. FFmpeg -decryption_key to decrypt in place + 3. Embed metadata tags (handled by the app's standard post-processing) +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import requests as http_requests + +from config.settings import config_manager +from core.amazon_client import AmazonClient, AmazonClientError +from core.download_plugins.base import DownloadSourcePlugin +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult +from utils.logging_config import get_logger + +logger = get_logger("amazon_download_client") + +# Quality / codec helpers +CODEC_PREFERENCE = ["flac", "opus", "eac3"] + +_CODEC_EXTENSIONS: Dict[str, str] = { + "flac": "flac", + "ogg_vorbis": "ogg", + "opus": "opus", + "eac3": "eac3", + "mp4": "m4a", + "aac": "m4a", + "mp3": "mp3", +} + +MIN_AUDIO_BYTES = 512 * 1024 # 512 KB — anything smaller is a broken stream + + +def _codec_key(codec: str) -> str: + return codec.lower().replace("-", "_").replace(" ", "_") + + +def _file_extension(codec: str) -> str: + return _CODEC_EXTENSIONS.get(_codec_key(codec), "bin") + + +def _quality_label(codec: str, sample_rate: Optional[int] = None) -> str: + ck = _codec_key(codec) + if ck == "flac": + if sample_rate and sample_rate > 48000: + return "Hi-Res" + return "Lossless" + return "Lossy" + + +class AmazonDownloadClient(DownloadSourcePlugin): + """DownloadSourcePlugin — Amazon Music via T2Tunes proxy.""" + + def __init__(self, download_path: Optional[str] = None) -> None: + if download_path is None: + download_path = config_manager.get("soulseek.download_path", "./downloads") + self.download_path = Path(download_path) + self.download_path.mkdir(parents=True, exist_ok=True) + + self._quality = config_manager.get("amazon_download.quality", "flac") + self._allow_fallback = config_manager.get("amazon_download.allow_fallback", True) + + self._client = AmazonClient(preferred_codec=self._quality) + self.session = http_requests.Session() + self.session.headers.update({ + "User-Agent": "SoulSync/1.0", + "Accept": "*/*", + }) + + self._engine: Any = None + self.shutdown_check: Any = None + + # ------------------------------------------------------------------ + # DownloadSourcePlugin — lifecycle + # ------------------------------------------------------------------ + + def set_engine(self, engine) -> None: + """Engine callback — wires the central thread worker + state store.""" + self._engine = engine + + def set_shutdown_check(self, check_callable) -> None: + self.shutdown_check = check_callable + + def is_configured(self) -> bool: + # T2Tunes has a public default instance; no credentials required. + # Return True unconditionally so the source shows as available. + return True + + async def check_connection(self) -> bool: + try: + return self._client.is_authenticated() + except Exception: + return False + + # ------------------------------------------------------------------ + # DownloadSourcePlugin — search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + timeout: Optional[int] = None, + progress_callback: Any = None, + ) -> Tuple[List[TrackResult], List[AlbumResult]]: + try: + items = self._client.search_raw(query, types="track,album") + except AmazonClientError as exc: + logger.warning(f"Amazon search failed for {query!r}: {exc}") + return [], [] + + track_results: List[TrackResult] = [] + album_map: Dict[str, AlbumResult] = {} + album_order: List[str] = [] + preferred = self._client.preferred_codec + + for item in items: + quality = _quality_label(preferred) + if item.is_track: + track_results.append(TrackResult( + username="amazon", + filename=f"{item.asin}||{item.artist_name} - {item.title}", + size=0, + bitrate=None, + duration=item.duration_seconds * 1000 if item.duration_seconds else None, + quality=quality, + free_upload_slots=999, + upload_speed=999_999, + queue_length=0, + artist=item.artist_name, + title=item.title, + album=item.album_name, + _source_metadata={ + "asin": item.asin, + "album_asin": item.album_asin, + "isrc": item.isrc, + }, + )) + elif item.is_album: + album_asin = item.album_asin or item.asin + if album_asin not in album_map: + placeholder = TrackResult( + username="amazon", + filename=f"{item.asin}||{item.artist_name} - {item.title}", + size=0, + bitrate=None, + duration=None, + quality=quality, + free_upload_slots=999, + upload_speed=999_999, + queue_length=0, + artist=item.artist_name, + title=item.title, + album=item.album_name, + ) + album_map[album_asin] = AlbumResult( + username="amazon", + album_path=album_asin, + album_title=item.album_name or item.title, + artist=item.artist_name, + track_count=0, + total_size=0, + tracks=[placeholder], + dominant_quality=quality, + ) + album_order.append(album_asin) + + return track_results, [album_map[k] for k in album_order] + + # ------------------------------------------------------------------ + # DownloadSourcePlugin — download dispatch + # ------------------------------------------------------------------ + + async def download( + self, + username: str, + filename: str, + file_size: int = 0, + ) -> Optional[str]: + if "||" not in filename: + logger.error(f"Invalid Amazon filename format (no '||'): {filename!r}") + return None + if self._engine is None: + raise RuntimeError( + "AmazonDownloadClient._engine is not set — " + "client not connected to download infrastructure" + ) + asin, display_name = filename.split("||", 1) + asin = asin.strip() + display_name = display_name.strip() + return self._engine.worker.dispatch( + source_name="amazon", + target_id=asin, + display_name=display_name, + original_filename=filename, + impl_callable=self._download_sync, + ) + + def _download_sync( + self, + download_id: str, + target_id: str, + display_name: str, + ) -> Optional[str]: + asin = str(target_id) + codecs = CODEC_PREFERENCE if self._allow_fallback else [self._quality] + for codec in codecs: + try: + streams = self._client.media_from_asin(asin, codec=codec) + except AmazonClientError as exc: + logger.warning(f"media_from_asin({asin!r}, {codec}) failed: {exc}") + continue + + stream = next( + (s for s in streams if s.streamable and s.stream_url), + None, + ) + if not stream: + logger.debug(f"No streamable result for {asin} at codec={codec}") + continue + + ext = _file_extension(stream.codec or codec) + safe = "".join( + ch if ch.isalnum() or ch in " -_." else "_" + for ch in display_name + )[:80] + # T2Tunes always serves audio in an encrypted MP4 container. + # The codec extension (.flac, .opus, .eac3) is only for the + # final decrypted output. + enc_ext = "mp4" if stream.decryption_key else ext + enc_path = self._unique_path(self.download_path / f"{safe}.enc.{enc_ext}") + out_path = self._unique_path(self.download_path / f"{safe}.{ext}") + + if self._engine is not None: + self._engine.update_record( + "amazon", download_id, {"state": "downloading", "progress": 0.0} + ) + try: + downloaded = self._stream_to_file(stream.stream_url, enc_path, download_id) + except Exception as exc: + logger.warning(f"Stream download failed for {asin} ({codec}): {exc}") + enc_path.unlink(missing_ok=True) + continue + + if downloaded < MIN_AUDIO_BYTES: + logger.warning( + f"File too small ({downloaded} B) for {asin} at {codec} — trying next" + ) + enc_path.unlink(missing_ok=True) + continue + + if stream.decryption_key: + if self._engine is not None: + self._engine.update_record( + "amazon", download_id, {"state": "decrypting", "progress": 1.0} + ) + try: + self._decrypt_with_ffmpeg(enc_path, out_path, stream.decryption_key) + enc_path.unlink(missing_ok=True) + except Exception as exc: + logger.error(f"Decryption failed for {asin} ({codec}): {exc}") + enc_path.unlink(missing_ok=True) + out_path.unlink(missing_ok=True) + continue + else: + enc_path.rename(out_path) + + final_size = out_path.stat().st_size + logger.info( + f"Amazon download complete ({codec}): {out_path} " + f"({final_size / (1024 * 1024):.1f} MB)" + ) + # Sync size == transferred so the download monitor's bytes-incomplete + # guard doesn't block post-processing. The throttled updates in + # _stream_to_file leave transferred < size after the last 0.5s tick; + # other streaming clients avoid this by not tracking bytes at all + # (size stays 0, the guard is skipped). Writing the final output size + # here restores parity. + if self._engine is not None: + self._engine.update_record( + "amazon", download_id, {'size': final_size, 'transferred': final_size} + ) + return str(out_path) + + logger.error(f"All codec tiers exhausted for '{display_name}' ({asin})") + return None + + def _decrypt_with_ffmpeg( + self, enc_path: Path, out_path: Path, hex_key: str + ) -> None: + """Decrypt a T2Tunes encrypted audio file using FFmpeg -decryption_key. + + T2Tunes uses CENC (Common Encryption) for DRM-protected tracks. + FFmpeg supports decryption via the -decryption_key flag when the + key is provided in hex. + """ + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + tools_dir = Path(__file__).parent.parent / "tools" + candidate = tools_dir / ("ffmpeg.exe" if os.name == "nt" else "ffmpeg") + if candidate.exists(): + ffmpeg = str(candidate) + else: + raise RuntimeError( + "ffmpeg is required for Amazon Music decryption. " + "Install ffmpeg and ensure it is on your PATH." + ) + cmd = [ + ffmpeg, + "-y", + "-hide_banner", + "-loglevel", "error", + "-decryption_key", hex_key, + "-i", str(enc_path), + "-map", "0:a:0", # extract first audio stream (FLAC/Opus/EAC3 inside MP4) + "-c", "copy", + str(out_path), + ] + logger.debug(f"Decrypting {enc_path.name} → {out_path.name}") + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(f"FFmpeg decryption failed (exit {result.returncode}): {stderr}") + + def _stream_to_file(self, url: str, out_path: Path, download_id: str) -> int: + resp = self.session.get(url, stream=True, timeout=60) + resp.raise_for_status() + + total = int(resp.headers.get("content-length", 0)) + downloaded = 0 + last_report = time.monotonic() + shutdown_triggered = False + + with out_path.open("wb") as fh: + for chunk in resp.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + if self.shutdown_check and self.shutdown_check(): + shutdown_triggered = True + break + fh.write(chunk) + downloaded += len(chunk) + now = time.monotonic() + if self._engine and now - last_report >= 0.5: + self._engine.update_record( + "amazon", + download_id, + { + "transferred": downloaded, + "size": total, + "progress": downloaded / total if total else 0.0, + }, + ) + last_report = now + + if shutdown_triggered: + out_path.unlink(missing_ok=True) + raise RuntimeError("Shutdown requested mid-download") + + return downloaded + + # ------------------------------------------------------------------ + # DownloadSourcePlugin — status interface + # ------------------------------------------------------------------ + + async def get_all_downloads(self) -> List[DownloadStatus]: + if self._engine is None: + return [] + return [ + self._record_to_status(record) + for record in self._engine.iter_records_for_source('amazon') + ] + + async def get_download_status(self, download_id: str) -> Optional[DownloadStatus]: + if self._engine is None: + return None + record = self._engine.get_record('amazon', download_id) + return self._record_to_status(record) if record is not None else None + + async def cancel_download( + self, + download_id: str, + username: Optional[str] = None, + remove: bool = False, + ) -> bool: + if self._engine is None: + return False + if self._engine.get_record('amazon', download_id) is None: + return False + self._engine.update_record('amazon', download_id, {'state': 'Cancelled'}) + if remove: + self._engine.remove_record('amazon', download_id) + return True + + async def clear_all_completed_downloads(self) -> bool: + if self._engine is None: + return True + terminal = {'Completed, Succeeded', 'Cancelled', 'Errored', 'Aborted'} + for record in list(self._engine.iter_records_for_source('amazon')): + if record.get('state') in terminal: + self._engine.remove_record('amazon', record['id']) + return True + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + @staticmethod + def _unique_path(path: Path) -> Path: + if not path.exists(): + return path + stem, suffix = path.stem, path.suffix + for i in range(1, 100): + candidate = path.with_name(f"{stem} ({i}){suffix}") + if not candidate.exists(): + return candidate + return path.with_name(f"{stem}_{uuid.uuid4().hex[:8]}{suffix}") + + @staticmethod + def _record_to_status(rec: Dict[str, Any]) -> DownloadStatus: + return DownloadStatus( + id=str(rec.get('id', '')), + filename=str(rec.get('filename', '')), + username='amazon', + state=str(rec.get('state', 'queued')), + progress=float(rec.get('progress', 0.0)), + size=int(rec.get('size', 0)), + transferred=int(rec.get('transferred', 0)), + speed=int(rec.get('speed', 0)), + time_remaining=rec.get('time_remaining'), + file_path=rec.get('file_path'), + ) diff --git a/core/amazon_worker.py b/core/amazon_worker.py new file mode 100644 index 00000000..59eaccb9 --- /dev/null +++ b/core/amazon_worker.py @@ -0,0 +1,566 @@ +import re +import threading +import time +from difflib import SequenceMatcher +from typing import Optional, Dict, Any +from datetime import datetime, timedelta +from utils.logging_config import get_logger +from database.music_database import MusicDatabase +from core.amazon_client import AmazonClient +from core.worker_utils import interruptible_sleep, set_album_api_track_count +from core.enrichment.manual_match_honoring import honor_stored_match + +logger = get_logger("amazon_worker") + + +class AmazonWorker: + """Background worker for enriching library artists, albums, and tracks with Amazon Music metadata.""" + + def __init__(self, database: MusicDatabase): + self.db = database + self.client = AmazonClient() + + self.running = False + self.paused = False + self.should_stop = False + self.thread = None + self._stop_event = threading.Event() + + self.current_item = None + + self.stats = { + 'matched': 0, + 'not_found': 0, + 'pending': 0, + 'errors': 0, + } + + self.retry_days = 30 + self.name_similarity_threshold = 0.80 + + logger.info("Amazon background worker initialized") + + def start(self): + if self.running: + logger.warning("Worker already running") + return + + self.running = True + self.should_stop = False + self._stop_event.clear() + self.thread = threading.Thread(target=self._run, daemon=True) + self.thread.start() + logger.info("Amazon background worker started") + + def stop(self): + if not self.running: + return + + logger.info("Stopping Amazon worker...") + self.should_stop = True + self.running = False + self._stop_event.set() + + if self.thread: + self.thread.join(timeout=1) + + logger.info("Amazon worker stopped") + + def pause(self): + if not self.running: + logger.warning("Worker not running, cannot pause") + return + self.paused = True + logger.info("Amazon worker paused") + + def resume(self): + if not self.running: + logger.warning("Worker not running, start it first") + return + self.paused = False + logger.info("Amazon worker resumed") + + def get_stats(self) -> Dict[str, Any]: + self.stats['pending'] = self._count_pending_items() + progress = self._get_progress_breakdown() + is_actually_running = self.running and (self.thread is not None and self.thread.is_alive()) + is_idle = is_actually_running and not self.paused and self.stats['pending'] == 0 and self.current_item is None + return { + 'enabled': True, + 'running': is_actually_running and not self.paused, + 'paused': self.paused, + 'idle': is_idle, + 'current_item': self.current_item, + 'stats': self.stats.copy(), + 'progress': progress, + } + + def _run(self): + logger.info("Amazon worker thread started") + while not self.should_stop: + try: + if self.paused: + interruptible_sleep(self._stop_event, 1) + continue + + self.current_item = None + item = self._get_next_item() + + if not item: + logger.debug("No pending items, sleeping...") + interruptible_sleep(self._stop_event, 10) + continue + + self.current_item = item + item_id = item.get('id') or item.get('artist_id') or item.get('album_id') + if item_id is None: + logger.warning(f"Skipping {item.get('type', 'unknown')} with NULL id: {item.get('name', '?')}") + continue + + self._process_item(item) + interruptible_sleep(self._stop_event, 2) + + except Exception as e: + logger.error(f"Error in worker loop: {e}") + interruptible_sleep(self._stop_event, 5) + + logger.info("Amazon worker thread finished") + + def _get_next_item(self) -> Optional[Dict[str, Any]]: + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + + # Priority 1: Unattempted artists + cursor.execute(""" + SELECT id, name FROM artists + WHERE amazon_match_status IS NULL AND id IS NOT NULL + ORDER BY id ASC LIMIT 1 + """) + row = cursor.fetchone() + if row: + return {'type': 'artist', 'id': row[0], 'name': row[1]} + + # Priority 2: Unattempted albums + cursor.execute(""" + SELECT a.id, a.title, ar.name AS artist_name, ar.amazon_id AS artist_amazon_id + FROM albums a + JOIN artists ar ON a.artist_id = ar.id + WHERE a.amazon_match_status IS NULL AND a.id IS NOT NULL + ORDER BY a.id ASC LIMIT 1 + """) + row = cursor.fetchone() + if row: + return {'type': 'album', 'id': row[0], 'name': row[1], 'artist': row[2], 'artist_amazon_id': row[3]} + + # Priority 3: Unattempted tracks + cursor.execute(""" + SELECT t.id, t.title, ar.name AS artist_name, ar.amazon_id AS artist_amazon_id + FROM tracks t + JOIN artists ar ON t.artist_id = ar.id + WHERE t.amazon_match_status IS NULL AND t.id IS NOT NULL + ORDER BY t.id ASC LIMIT 1 + """) + row = cursor.fetchone() + if row: + return {'type': 'track', 'id': row[0], 'name': row[1], 'artist': row[2], 'artist_amazon_id': row[3]} + + not_found_cutoff = datetime.now() - timedelta(days=self.retry_days) + + # Priority 4: Retry not_found artists + cursor.execute(""" + SELECT id, name FROM artists + WHERE amazon_match_status = 'not_found' AND amazon_last_attempted < ? + ORDER BY amazon_last_attempted ASC LIMIT 1 + """, (not_found_cutoff,)) + row = cursor.fetchone() + if row: + logger.info(f"Retrying artist '{row[1]}' (last attempted before cutoff)") + return {'type': 'artist', 'id': row[0], 'name': row[1]} + + # Priority 5: Retry not_found albums + cursor.execute(""" + SELECT a.id, a.title, ar.name AS artist_name, ar.amazon_id AS artist_amazon_id + FROM albums a + JOIN artists ar ON a.artist_id = ar.id + WHERE a.amazon_match_status = 'not_found' AND a.amazon_last_attempted < ? + ORDER BY a.amazon_last_attempted ASC LIMIT 1 + """, (not_found_cutoff,)) + row = cursor.fetchone() + if row: + return {'type': 'album', 'id': row[0], 'name': row[1], 'artist': row[2], 'artist_amazon_id': row[3]} + + # Priority 6: Retry not_found tracks + cursor.execute(""" + SELECT t.id, t.title, ar.name AS artist_name, ar.amazon_id AS artist_amazon_id + FROM tracks t + JOIN artists ar ON t.artist_id = ar.id + WHERE t.amazon_match_status = 'not_found' AND t.amazon_last_attempted < ? + ORDER BY t.amazon_last_attempted ASC LIMIT 1 + """, (not_found_cutoff,)) + row = cursor.fetchone() + if row: + return {'type': 'track', 'id': row[0], 'name': row[1], 'artist': row[2], 'artist_amazon_id': row[3]} + + return None + + except Exception as e: + logger.error(f"Error getting next item: {e}") + return None + finally: + if conn: + conn.close() + + def _normalize_name(self, name: str) -> str: + name = name.lower().strip() + name = re.sub(r'\s+[-–—]\s+.*$', '', name) + name = re.sub(r'\s*\(.*?\)\s*', ' ', name) + name = re.sub(r'[^\w\s]', '', name) + name = re.sub(r'\s+', ' ', name).strip() + return name + + def _name_matches(self, query_name: str, result_name: str) -> bool: + norm_query = self._normalize_name(query_name) + norm_result = self._normalize_name(result_name) + similarity = SequenceMatcher(None, norm_query, norm_result).ratio() + logger.debug(f"Name similarity: '{query_name}' vs '{result_name}' = {similarity:.2f}") + return similarity >= self.name_similarity_threshold + + def _process_item(self, item: Dict[str, Any]): + try: + item_type = item['type'] + item_id = item['id'] + item_name = item['name'] + logger.debug(f"Processing {item_type} #{item_id}: {item_name}") + + if item_type == 'artist': + self._process_artist(item_id, item_name) + elif item_type == 'album': + self._process_album(item_id, item_name, item.get('artist', ''), item) + elif item_type == 'track': + self._process_track(item_id, item_name, item.get('artist', ''), item) + + except Exception as e: + logger.error(f"Error processing {item['type']} #{item['id']}: {e}") + self.stats['errors'] += 1 + try: + self._mark_status(item['type'], item['id'], 'error') + except Exception as e2: + logger.error(f"Error updating item status: {e2}") + + def _get_existing_id(self, entity_type: str, entity_id: int) -> Optional[str]: + table_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'} + table = table_map.get(entity_type) + if not table: + return None + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cursor.execute(f"SELECT amazon_id FROM {table} WHERE id = ?", (entity_id,)) + row = cursor.fetchone() + return row[0] if row and row[0] else None + except Exception: + return None + finally: + if conn: + conn.close() + + def _process_artist(self, artist_id: int, artist_name: str): + existing_id = self._get_existing_id('artist', artist_id) + if existing_id: + logger.debug(f"Preserving existing Amazon ID for artist '{artist_name}': {existing_id}") + return + + results = self.client.search_artists(artist_name, limit=5) + if results: + result = results[0] + if self._name_matches(artist_name, result.name): + self._update_artist(artist_id, result) + self.stats['matched'] += 1 + logger.info(f"Matched artist '{artist_name}' -> Amazon ID: {result.id}") + else: + self._mark_status('artist', artist_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"Name mismatch for artist '{artist_name}' (got '{result.name}')") + else: + self._mark_status('artist', artist_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"No match for artist '{artist_name}'") + + def _refresh_album_via_stored_id(self, album_id, stored_id, api_data): + self._update_album(album_id, api_data, stored_id) + + def _refresh_track_via_stored_id(self, track_id, stored_id, api_data): + self._update_track(track_id, api_data, stored_id) + + def _process_album(self, album_id: int, album_name: str, artist_name: str, item: Dict[str, Any]): + if honor_stored_match( + db=self.db, entity_table='albums', entity_id=album_id, + id_column='amazon_id', + client_fetch_fn=lambda asin: self.client.get_album(asin, include_tracks=False), + on_match_fn=self._refresh_album_via_stored_id, + log_prefix='Amazon', + ): + self.stats['matched'] += 1 + return + + query = f"{artist_name} {album_name}" + results = self.client.search_albums(query, limit=10) + if results: + result = results[0] + if self._name_matches(album_name, result.name): + full_album = None + if result.id: + try: + full_album = self.client.get_album(result.id, include_tracks=False) + except Exception as e: + logger.warning(f"Failed to fetch full album '{album_name}' (ASIN: {result.id}): {e}") + + if full_album is None: + self._mark_status('album', album_id, 'error') + self.stats['errors'] += 1 + logger.warning(f"Album '{album_name}' matched but full details unavailable, will retry") + return + + self._update_album(album_id, full_album, result.id) + self.stats['matched'] += 1 + logger.info(f"Matched album '{album_name}' -> Amazon ASIN: {result.id}") + else: + self._mark_status('album', album_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"Name mismatch for album '{album_name}' (got '{result.name}')") + else: + self._mark_status('album', album_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"No match for album '{album_name}'") + + def _process_track(self, track_id: int, track_name: str, artist_name: str, item: Dict[str, Any]): + if honor_stored_match( + db=self.db, entity_table='tracks', entity_id=track_id, + id_column='amazon_id', + client_fetch_fn=self.client.get_track_details, + on_match_fn=self._refresh_track_via_stored_id, + log_prefix='Amazon', + ): + self.stats['matched'] += 1 + return + + query = f"{artist_name} {track_name}" + results = self.client.search_tracks(query, limit=10) + if results: + result = results[0] + if self._name_matches(track_name, result.name): + full_track = None + if result.id: + try: + full_track = self.client.get_track_details(result.id) + except Exception as e: + logger.warning(f"Failed to fetch full track '{track_name}' (ASIN: {result.id}): {e}") + + if full_track is None: + self._mark_status('track', track_id, 'error') + self.stats['errors'] += 1 + logger.warning(f"Track '{track_name}' matched but full details unavailable, will retry") + return + + self._update_track(track_id, full_track, result.id) + self.stats['matched'] += 1 + logger.info(f"Matched track '{track_name}' -> Amazon ASIN: {result.id}") + else: + self._mark_status('track', track_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"Name mismatch for track '{track_name}' (got '{result.name}')") + else: + self._mark_status('track', track_id, 'not_found') + self.stats['not_found'] += 1 + logger.debug(f"No match for track '{track_name}'") + + def _update_artist(self, artist_id: int, result): + """Store Amazon metadata for an artist. ``result`` is an Artist dataclass.""" + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + + cursor.execute(""" + UPDATE artists SET + amazon_id = ?, + amazon_match_status = 'matched', + amazon_last_attempted = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (str(result.id), artist_id)) + + # Backfill thumb_url from album cover stand-in when artist has no image + image_url = result.image_url + if not image_url: + try: + image_url = self.client._get_artist_image_from_albums(result.id) + except Exception as exc: + logger.debug("Artist image from albums failed for %s: %s", result.id, exc) + if image_url: + cursor.execute(""" + UPDATE artists SET thumb_url = ? + WHERE id = ? AND (thumb_url IS NULL OR thumb_url = '') + """, (image_url, artist_id)) + + conn.commit() + + except Exception as e: + logger.error(f"Error updating artist #{artist_id} with Amazon data: {e}") + raise + finally: + if conn: + conn.close() + + def _update_album(self, album_id: int, full_data: Dict[str, Any], asin: str): + """Store Amazon metadata for an album. ``full_data`` is a get_album() dict.""" + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + + cursor.execute(""" + UPDATE albums SET + amazon_id = ?, + amazon_match_status = 'matched', + amazon_last_attempted = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (asin, album_id)) + + # Backfill label when missing + label = full_data.get('label') + if label: + cursor.execute(""" + UPDATE albums SET label = ? + WHERE id = ? AND (label IS NULL OR label = '') + """, (label, album_id)) + + # Backfill thumb_url + images = full_data.get('images') or [] + thumb_url = images[0].get('url') if images else None + if thumb_url: + cursor.execute(""" + UPDATE albums SET thumb_url = ? + WHERE id = ? AND (thumb_url IS NULL OR thumb_url = '') + """, (thumb_url, album_id)) + + # Cache authoritative track count for completeness repair + total_tracks = full_data.get('total_tracks') or ( + full_data.get('tracks', {}).get('total') if isinstance(full_data.get('tracks'), dict) else None + ) + set_album_api_track_count(cursor, album_id, total_tracks) + + conn.commit() + + except Exception as e: + logger.error(f"Error updating album #{album_id} with Amazon data: {e}") + raise + finally: + if conn: + conn.close() + + def _update_track(self, track_id: int, full_data: Dict[str, Any], asin: str): + """Store Amazon metadata for a track. ``full_data`` is a get_track_details() dict.""" + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + + cursor.execute(""" + UPDATE tracks SET + amazon_id = ?, + amazon_match_status = 'matched', + amazon_last_attempted = CURRENT_TIMESTAMP + WHERE id = ? + """, (asin, track_id)) + + conn.commit() + + except Exception as e: + logger.error(f"Error updating track #{track_id} with Amazon data: {e}") + raise + finally: + if conn: + conn.close() + + def _mark_status(self, entity_type: str, entity_id: int, status: str): + table_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'} + table = table_map.get(entity_type) + if not table: + logger.error(f"Unknown entity type: {entity_type}") + return + + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cursor.execute(f""" + UPDATE {table} SET + amazon_match_status = ?, + amazon_last_attempted = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (status, entity_id)) + conn.commit() + except Exception as e: + logger.error(f"Error marking {entity_type} #{entity_id} status: {e}") + finally: + if conn: + conn.close() + + def _count_pending_items(self) -> int: + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT + (SELECT COUNT(*) FROM artists WHERE amazon_match_status IS NULL AND id IS NOT NULL) + + (SELECT COUNT(*) FROM albums WHERE amazon_match_status IS NULL AND id IS NOT NULL) + + (SELECT COUNT(*) FROM tracks WHERE amazon_match_status IS NULL AND id IS NOT NULL) + AS pending + """) + row = cursor.fetchone() + return row[0] if row else 0 + except Exception as e: + logger.error(f"Error counting pending items: {e}") + return 0 + finally: + if conn: + conn.close() + + def _get_progress_breakdown(self) -> Dict[str, Dict[str, int]]: + conn = None + try: + conn = self.db._get_connection() + cursor = conn.cursor() + progress = {} + + for table in ('artists', 'albums', 'tracks'): + cursor.execute(f""" + SELECT + COUNT(*) AS total, + SUM(CASE WHEN amazon_match_status IS NOT NULL THEN 1 ELSE 0 END) AS processed + FROM {table} + """) + row = cursor.fetchone() + if row: + total, processed = row[0], row[1] or 0 + progress[table] = { + 'matched': processed, + 'total': total, + 'percent': int((processed / total * 100) if total > 0 else 0), + } + + return progress + + except Exception as e: + logger.error(f"Error getting progress breakdown: {e}") + return {} + finally: + if conn: + conn.close() diff --git a/core/api_call_tracker.py b/core/api_call_tracker.py index 7fb561e1..95e1e18e 100644 --- a/core/api_call_tracker.py +++ b/core/api_call_tracker.py @@ -30,6 +30,7 @@ RATE_LIMITS = { 'tidal': 120, # MIN_API_INTERVAL=0.5s → ~120/min 'qobuz': 60, # Variable throttle, ~60/min estimate 'discogs': 60, # MIN_API_INTERVAL=1.0s with auth → ~60/min + 'amazon': 120, # MIN_API_INTERVAL=0.5s → ~120/min (T2Tunes proxy) } # Display names for UI @@ -44,12 +45,13 @@ SERVICE_LABELS = { 'tidal': 'Tidal', 'qobuz': 'Qobuz', 'discogs': 'Discogs', + 'amazon': 'Amazon Music', } # Display order SERVICE_ORDER = [ 'spotify', 'itunes', 'deezer', 'lastfm', 'genius', - 'musicbrainz', 'audiodb', 'tidal', 'qobuz', 'discogs', + 'musicbrainz', 'audiodb', 'tidal', 'qobuz', 'discogs', 'amazon', ] diff --git a/core/artist_source_detail.py b/core/artist_source_detail.py index c376052e..35b3e4de 100644 --- a/core/artist_source_detail.py +++ b/core/artist_source_detail.py @@ -43,6 +43,7 @@ def build_source_only_artist_detail( deezer_client: Optional[Any] = None, itunes_client: Optional[Any] = None, discogs_client: Optional[Any] = None, + amazon_client: Optional[Any] = None, lastfm_api_key: Optional[str] = None, ) -> Tuple[Dict[str, Any], int]: """Build the artist-detail payload for a source-only artist. @@ -84,6 +85,12 @@ def build_source_only_artist_detail( dc_artist = discogs_client.get_artist(artist_id) if dc_artist: source_genres = dc_artist.get("genres") or [] + elif source == "amazon" and amazon_client is not None: + az_artist = amazon_client.get_artist(resolved_name or artist_id) + if az_artist: + source_genres = az_artist.get("genres") or [] + if not image_url and az_artist.get("images"): + image_url = az_artist["images"][0].get("url") except Exception as e: logger.debug(f"Source-side artist info lookup failed for {source}:{artist_id}: {e}") diff --git a/core/artist_source_lookup.py b/core/artist_source_lookup.py index b965d4cf..507d09e6 100644 --- a/core/artist_source_lookup.py +++ b/core/artist_source_lookup.py @@ -25,7 +25,7 @@ logger = logging.getLogger("artist_source_lookup") SOURCE_ONLY_ARTIST_SOURCES = frozenset({ - "spotify", "itunes", "deezer", "discogs", "hydrabase", "musicbrainz", + "spotify", "itunes", "deezer", "discogs", "hydrabase", "musicbrainz", "amazon", }) @@ -36,6 +36,7 @@ SOURCE_ID_FIELD = { "discogs": "discogs_id", "hydrabase": "soul_id", "musicbrainz": "musicbrainz_id", + "amazon": "amazon_id", } diff --git a/core/artists/map.py b/core/artists/map.py index 5e6f130a..c2ceb0c5 100644 --- a/core/artists/map.py +++ b/core/artists/map.py @@ -20,10 +20,14 @@ logger = logging.getLogger(__name__) def get_current_profile_id() -> int: - """Mirror of web_server.get_current_profile_id — uses Flask g.""" + """Mirror of web_server.get_current_profile_id — uses Flask g. + + Catches RuntimeError too because reading `g` outside a request + context raises that (not AttributeError) — happens when this is + called from background threads (sync, automation, scanners).""" try: return g.profile_id - except AttributeError: + except (AttributeError, RuntimeError): return 1 diff --git a/core/automation/__init__.py b/core/automation/__init__.py index 7841a13e..043b8173 100644 --- a/core/automation/__init__.py +++ b/core/automation/__init__.py @@ -1,7 +1,11 @@ -"""Automation API + progress tracking helpers package. +"""Automation API + progress + handlers package. -Lifted from web_server.py /api/automations/* routes and progress -emitters. The action handler registration (`_register_automation_handlers`) -stays in web_server.py because each handler closure is tightly coupled -to other application features. +Lifted from web_server.py: + - `/api/automations/*` route helpers → `api.py` + - block library used by the trigger/action UI → `blocks.py` + - progress tracker (init / update / finish) → `progress.py` + - cross-handler signal bus → `signals.py` + - per-action handler functions → `handlers/` subpackage (with + `deps.py` defining the dependency-injection surface so handlers + stay testable in isolation) """ diff --git a/core/automation/blocks.py b/core/automation/blocks.py index c84d1b37..2697af26 100644 --- a/core/automation/blocks.py +++ b/core/automation/blocks.py @@ -146,6 +146,15 @@ ACTIONS: list[dict] = [ {"key": "all", "type": "checkbox", "label": "Process all mirrored playlists", "default": False}, {"key": "skip_wishlist", "type": "checkbox", "label": "Skip wishlist processing", "default": False}, ]}, + {"type": "personalized_pipeline", "label": "Personalized Playlist Pipeline", "icon": "sparkles", + "description": "Sync personalized / discover-page playlists (Hidden Gems, Time Machine, Fresh Tape, etc.) to your media server + queue missing tracks for download.", + "available": True, + "config_fields": [ + {"key": "kinds", "type": "personalized_playlist_select", "label": "Playlists to sync", + "description": "Multi-select: Hidden Gems, Discovery Shuffle, Time Machine (per decade), Genre playlists, Fresh Tape, The Archives, Seasonal Mix (per season)"}, + {"key": "refresh_first", "type": "checkbox", "label": "Refresh playlists before sync (regenerate snapshots)", "default": False}, + {"key": "skip_wishlist", "type": "checkbox", "label": "Skip wishlist processing", "default": False}, + ]}, {"type": "notify_only", "label": "Notify Only", "icon": "bell", "description": "No action — just send notification", "available": True}, # Phase 3 actions {"type": "start_database_update", "label": "Update Database", "icon": "database", diff --git a/core/automation/deps.py b/core/automation/deps.py new file mode 100644 index 00000000..a521a496 --- /dev/null +++ b/core/automation/deps.py @@ -0,0 +1,146 @@ +"""Dependency-injection surface for automation handlers. + +Each handler in ``core.automation.handlers`` is a top-level pure +function that accepts ``(config: dict, deps: AutomationDeps)`` instead +of reaching for module-level globals in ``web_server``. The deps +namespace bundles every callable, client, and mutable-state container +the handlers need. + +Construction happens once at app startup in ``web_server.py``: + + from core.automation.deps import AutomationDeps, AutomationState + state = AutomationState() + deps = AutomationDeps( + engine=automation_engine, + state=state, + get_database=get_database, + spotify_client=spotify_client, + ... + ) + register_all(deps) + +Tests construct a fake ``AutomationDeps`` with stub callables — every +handler is then exercisable without spinning up Flask, the DB, or +the real media-server clients. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + + +@dataclass +class AutomationState: + """Mutable flags shared across handler invocations. + + Pre-refactor each was a ``global`` or ``nonlocal`` variable inside + the registration closure. Lifted here so handlers + their guards + can read/write a single object instead of importing globals. + + All mutations should hold ``lock``; the helper methods below do + so for the common get/set patterns. + """ + + scan_library_automation_id: Optional[str] = None + db_update_automation_id: Optional[str] = None + pipeline_running: bool = False + lock: threading.Lock = field(default_factory=threading.Lock) + + def is_scan_library_active(self) -> bool: + with self.lock: + return self.scan_library_automation_id is not None + + def is_pipeline_running(self) -> bool: + with self.lock: + return self.pipeline_running + + def set_scan_library_id(self, automation_id: Optional[str]) -> None: + with self.lock: + self.scan_library_automation_id = automation_id + + def set_pipeline_running(self, value: bool) -> None: + with self.lock: + self.pipeline_running = value + + +@dataclass +class AutomationDeps: + """Bundle of every callable + client an automation handler may need. + + Add fields as new handlers are extracted. Every field is required + at construction (no defaults) so a missing dep fails loudly at + startup, not silently mid-handler. + """ + + # --- Engine + shared state --- + engine: Any # AutomationEngine instance + state: AutomationState + config_manager: Any # config.settings.ConfigManager singleton + update_progress: Callable[..., None] # _update_automation_progress + logger: Any # module-level logger from utils.logging_config + + # --- Service clients (each may be None depending on user config) --- + get_database: Callable[[], Any] # late-binding so tests don't need DB + spotify_client: Any + tidal_client: Any + web_scan_manager: Any + + # --- Background-task entry points --- + process_wishlist_automatically: Callable[..., Any] + process_watchlist_scan_automatically: Callable[..., Any] + is_wishlist_actually_processing: Callable[[], bool] + is_watchlist_actually_scanning: Callable[[], bool] + get_watchlist_scan_state: Callable[[], dict] # accessor returns the live mutable dict + + # --- Playlist pipeline entry points --- + run_playlist_discovery_worker: Callable[..., Any] + run_sync_task: Callable[..., Any] + load_sync_status_file: Callable[[], dict] + get_deezer_client: Callable[[], Any] + parse_youtube_playlist: Callable[[str], Any] + get_sync_states: Callable[[], dict] # accessor returns the live dict shared with the sync UI + + # --- Database update + quality scanner (shared state + executors) --- + set_db_update_automation_id: Callable[[Optional[str]], None] # syncs the legacy `_db_update_automation_id` global so the live DB-update progress callbacks (which still read the global directly) keep firing for the active automation + get_db_update_state: Callable[[], dict] + db_update_lock: Any # threading.Lock + db_update_executor: Any # ThreadPoolExecutor + run_db_update_task: Callable[..., Any] + run_deep_scan_task: Callable[..., Any] + get_duplicate_cleaner_state: Callable[[], dict] + duplicate_cleaner_lock: Any + duplicate_cleaner_executor: Any + run_duplicate_cleaner: Callable[..., Any] + get_quality_scanner_state: Callable[[], dict] + quality_scanner_lock: Any + quality_scanner_executor: Any + run_quality_scanner: Callable[..., Any] + + # --- Download orchestrator + queue accessors --- + download_orchestrator: Any + run_async: Callable[..., Any] + tasks_lock: Any + get_download_batches: Callable[[], dict] + get_download_tasks: Callable[[], dict] + sweep_empty_download_directories: Callable[[], int] + get_staging_path: Callable[[], str] + + # --- Maintenance helpers --- + docker_resolve_path: Callable[[str], str] + get_current_profile_id: Callable[[], int] + get_watchlist_scanner: Callable[[Any], Any] + get_app: Callable[[], Any] # Flask app for test_client (beatport refresh) + get_beatport_data_cache: Callable[[], dict] + + # --- Progress + history callbacks (used by register_all to wire + # the engine's progress callback hooks). --- + init_automation_progress: Callable[..., Any] + record_progress_history: Callable[..., Any] + + # --- Personalized playlist pipeline --- + # Lazy builder so the pipeline handler can construct a fresh + # PersonalizedPlaylistManager per run (cheap accessors inside, + # no caching needed yet). + build_personalized_manager: Callable[[], Any] diff --git a/core/automation/handlers/__init__.py b/core/automation/handlers/__init__.py new file mode 100644 index 00000000..7a02f4d4 --- /dev/null +++ b/core/automation/handlers/__init__.py @@ -0,0 +1,64 @@ +"""Per-action automation handlers. + +Each module in this subpackage exposes one top-level handler function +(or a small cluster of related handlers) of the form:: + + def auto_(config: dict, deps: AutomationDeps) -> dict + +The ``register_all`` helper in :mod:`registration` wires every handler +to the engine in one place. ``web_server.py`` calls +``register_all(deps)`` once at startup. +""" + +from core.automation.handlers.process_wishlist import auto_process_wishlist +from core.automation.handlers.scan_watchlist import auto_scan_watchlist +from core.automation.handlers.scan_library import auto_scan_library +from core.automation.handlers.refresh_mirrored import auto_refresh_mirrored +from core.automation.handlers.sync_playlist import auto_sync_playlist +from core.automation.handlers.discover_playlist import auto_discover_playlist +from core.automation.handlers.playlist_pipeline import auto_playlist_pipeline +from core.automation.handlers.personalized_pipeline import auto_personalized_pipeline +from core.automation.handlers.database_update import auto_start_database_update, auto_deep_scan_library +from core.automation.handlers.duplicate_cleaner import auto_run_duplicate_cleaner +from core.automation.handlers.quality_scanner import auto_start_quality_scan +from core.automation.handlers.maintenance import ( + auto_clear_quarantine, + auto_cleanup_wishlist, + auto_update_discovery_pool, + auto_backup_database, + auto_refresh_beatport_cache, +) +from core.automation.handlers.download_cleanup import ( + auto_clean_search_history, + auto_clean_completed_downloads, + auto_full_cleanup, +) +from core.automation.handlers.run_script import auto_run_script +from core.automation.handlers.search_and_download import auto_search_and_download +from core.automation.handlers.registration import register_all + +__all__ = [ + 'auto_process_wishlist', + 'auto_scan_watchlist', + 'auto_scan_library', + 'auto_refresh_mirrored', + 'auto_sync_playlist', + 'auto_discover_playlist', + 'auto_playlist_pipeline', + 'auto_personalized_pipeline', + 'auto_start_database_update', + 'auto_deep_scan_library', + 'auto_run_duplicate_cleaner', + 'auto_start_quality_scan', + 'auto_clear_quarantine', + 'auto_cleanup_wishlist', + 'auto_update_discovery_pool', + 'auto_backup_database', + 'auto_refresh_beatport_cache', + 'auto_clean_search_history', + 'auto_clean_completed_downloads', + 'auto_full_cleanup', + 'auto_run_script', + 'auto_search_and_download', + 'register_all', +] diff --git a/core/automation/handlers/_pipeline_shared.py b/core/automation/handlers/_pipeline_shared.py new file mode 100644 index 00000000..c7427f8a --- /dev/null +++ b/core/automation/handlers/_pipeline_shared.py @@ -0,0 +1,201 @@ +"""Shared helpers between mirrored + personalized playlist pipelines. + +Both pipelines end in the same shape: +1. SYNC each playlist to the active media server. +2. WISHLIST: trigger the wishlist processor for missing tracks. + +The differing prefix (mirrored = REFRESH external sources + DISCOVER +metadata; personalized = SNAPSHOT manager-backed playlists) is owned +by each pipeline. This module owns the SYNC + WISHLIST tail so both +pipelines stay consistent + DRY. +""" + +from __future__ import annotations + +import time +from typing import Any, Callable, Dict, List, Optional + +from core.automation.deps import AutomationDeps + + +# Per-playlist sync poll cap (mirrored side already used this). +_SYNC_PER_PLAYLIST_TIMEOUT_SECONDS = 600 +# Sync-status final-state markers. +_SYNC_TERMINAL_STATUSES = ('finished', 'complete', 'error', 'failed') + + +def run_sync_and_wishlist( + deps: AutomationDeps, + automation_id: Optional[str], + playlists: List[Dict[str, Any]], + *, + sync_one_fn: Callable[[Dict[str, Any]], Dict[str, Any]], + sync_id_for_fn: Callable[[Dict[str, Any]], str], + skip_wishlist: bool = False, + progress_start: int = 56, + progress_end: int = 85, + sync_phase_label: str = 'Phase: Syncing to server...', + sync_phase_start_log: str = 'Sync', + wishlist_phase_label: str = 'Phase: Processing wishlist...', + wishlist_phase_start_log: str = 'Wishlist', +) -> Dict[str, int]: + """Run the SYNC + WISHLIST tail of a playlist pipeline. + + The caller supplies: + - ``playlists``: list of playlist payload dicts. Each must have at + least a ``name`` key (used in progress logs). The shape beyond + ``name`` is opaque to the helper — ``sync_one_fn`` receives the + payload and returns a sync_result dict. + - ``sync_one_fn(payload) -> sync_result``: launches sync for one + playlist. Result dict must carry ``status`` ∈ ``('started', + 'skipped', 'error')`` and may carry ``reason``. + - ``sync_id_for_fn(payload) -> str``: returns the sync-state key + the helper polls on (so we can wait for the background sync + thread to complete + read the matched_tracks count). + + Returns ``{'synced': int, 'skipped': int, 'errors': int, + 'wishlist_queued': int}`` so the caller can stitch it into its + final status. + """ + deps.update_progress( + automation_id, + progress=progress_start, + phase=sync_phase_label, + log_line=sync_phase_start_log, + log_type='info', + ) + + total_synced = 0 + total_skipped = 0 + sync_errors = 0 + sync_states = deps.get_sync_states() + n_playlists = max(1, len(playlists)) + progress_span = max(1, progress_end - progress_start - 1) + + for pl_idx, pl in enumerate(playlists): + pl_name = pl.get('name', '') + sync_result = sync_one_fn(pl) + sync_status = sync_result.get('status', '') + + if sync_status == 'started': + sync_id = sync_id_for_fn(pl) + sync_poll_start = time.time() + while time.time() - sync_poll_start < _SYNC_PER_PLAYLIST_TIMEOUT_SECONDS: + if (sync_id in sync_states + and sync_states[sync_id].get('status') in _SYNC_TERMINAL_STATUSES): + break + time.sleep(2) + elapsed = int(time.time() - sync_poll_start) + sub_progress = progress_start + 1 + ((pl_idx + 1) / n_playlists) * progress_span + deps.update_progress( + automation_id, + progress=min(int(sub_progress), progress_end - 1), + phase=f'{sync_phase_label.rstrip(".")} — "{pl_name}" ({elapsed}s)', + ) + + ss = sync_states.get(sync_id, {}) + ss_result = ss.get('result', ss.get('progress', {})) + matched = ss_result.get('matched_tracks', 0) if isinstance(ss_result, dict) else 0 + total_synced += int(matched) if matched else 0 + deps.update_progress( + automation_id, + log_line=f'Synced "{pl_name}": {matched} tracks matched', + log_type='success', + ) + + elif sync_status == 'skipped': + total_skipped += 1 + reason = sync_result.get('reason', 'unchanged') + deps.update_progress( + automation_id, + log_line=f'Skipped "{pl_name}": {reason}', + log_type='skip', + ) + elif sync_status == 'error': + sync_errors += 1 + deps.update_progress( + automation_id, + log_line=f'Sync error "{pl_name}": {sync_result.get("reason", "unknown")}', + log_type='error', + ) + + deps.update_progress( + automation_id, + progress=progress_end, + phase=f'{sync_phase_label.rstrip(".")} complete', + log_line=f'Sync done: {total_synced} matched, {total_skipped} skipped, {sync_errors} errors', + log_type='success' if sync_errors == 0 else 'warning', + ) + + wishlist_queued = run_wishlist_phase( + deps, automation_id, + skip=skip_wishlist, + progress_pct=progress_end + 1, + wishlist_phase_label=wishlist_phase_label, + wishlist_phase_start_log=wishlist_phase_start_log, + ) + + return { + 'synced': total_synced, + 'skipped': total_skipped, + 'errors': sync_errors, + 'wishlist_queued': wishlist_queued, + } + + +def run_wishlist_phase( + deps: AutomationDeps, + automation_id: Optional[str], + *, + skip: bool, + progress_pct: int, + wishlist_phase_label: str = 'Phase: Processing wishlist...', + wishlist_phase_start_log: str = 'Wishlist', +) -> int: + """Trigger the wishlist processor unless skipped or already running. + + Returns 1 when the processor was triggered, 0 otherwise. Errors are + logged but never raised — wishlist failure should not abort the + pipeline.""" + if skip: + deps.update_progress( + automation_id, + progress=progress_pct, + log_line=f'{wishlist_phase_start_log}: skipped (disabled)', + log_type='skip', + ) + return 0 + + deps.update_progress( + automation_id, + progress=progress_pct, + phase=wishlist_phase_label, + log_line=wishlist_phase_start_log, + log_type='info', + ) + + try: + if not deps.is_wishlist_actually_processing(): + deps.process_wishlist_automatically(automation_id=None) + deps.update_progress( + automation_id, + log_line='Wishlist processing triggered', + log_type='success', + ) + return 1 + deps.update_progress( + automation_id, + log_line='Wishlist already running — skipped', + log_type='skip', + ) + return 0 + except Exception as e: # noqa: BLE001 — wishlist failure must never abort pipeline + deps.update_progress( + automation_id, + log_line=f'Wishlist error: {e}', + log_type='warning', + ) + return 0 + + +__all__ = ['run_sync_and_wishlist', 'run_wishlist_phase'] diff --git a/core/automation/handlers/database_update.py b/core/automation/handlers/database_update.py new file mode 100644 index 00000000..a9d51036 --- /dev/null +++ b/core/automation/handlers/database_update.py @@ -0,0 +1,136 @@ +"""Automation handlers: ``start_database_update`` and +``deep_scan_library`` actions. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_start_database_update`` and ``_auto_deep_scan_library`` +closures). Both share the same ``db_update_state`` / executor / lock +infrastructure -- the only difference is which task they submit +(``run_db_update_task`` vs ``run_deep_scan_task``). + +Pattern: pre-set state to running, submit task to executor, then +poll the state dict until it transitions away from ``running``. +Stall-detection emits a warning every 10 minutes when progress +hasn't budged. 2-hour outer timeout caps the worst case. +""" + +from __future__ import annotations + +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +_TIMEOUT_SECONDS = 7200 # 2 hours — covers the worst large-library case +_STALL_WARNING_SECONDS = 600 # 10 minutes without progress = stall +_POLL_INTERVAL_SECONDS = 3 +_INITIAL_DELAY_SECONDS = 1 + + +def auto_start_database_update(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run a full or incremental DB update via ``run_db_update_task``.""" + return _run_with_progress( + config, deps, + task=deps.run_db_update_task, + task_args=(config.get('full_refresh', False), deps.config_manager.get_active_media_server()), + initial_phase='Initializing...', + stall_label='Database update', + finished_extras=lambda: {'full_refresh': str(config.get('full_refresh', False))}, + timeout_label='Database update timed out after 2 hours', + ) + + +def auto_deep_scan_library(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run a deep library scan via ``run_deep_scan_task``.""" + return _run_with_progress( + config, deps, + task=deps.run_deep_scan_task, + task_args=(deps.config_manager.get_active_media_server(),), + initial_phase='Deep scan: Initializing...', + stall_label='Deep scan', + finished_extras=lambda: {}, + timeout_label='Deep scan timed out after 2 hours', + ) + + +def _run_with_progress( + config: Dict[str, Any], + deps: AutomationDeps, + *, + task, + task_args: tuple, + initial_phase: str, + stall_label: str, + finished_extras, + timeout_label: str, +) -> Dict[str, Any]: + """Shared poll-and-wait body for both DB-update handlers.""" + automation_id = config.get('_automation_id') + state = deps.get_db_update_state() + if state.get('status') == 'running': + return {'status': 'skipped', 'reason': 'Database update already running'} + deps.state.db_update_automation_id = automation_id + # Sync legacy module global so the DB-update progress callbacks + # (still living in web_server.py) emit against this automation. + deps.set_db_update_automation_id(automation_id) + + with deps.db_update_lock: + state.update({ + 'status': 'running', 'phase': initial_phase, + 'progress': 0, 'current_item': '', 'processed': 0, 'total': 0, + 'error_message': '', + }) + deps.db_update_executor.submit(task, *task_args) + + # Monitor progress (callbacks handle card updates, we just block until done). + time.sleep(_INITIAL_DELAY_SECONDS) + poll_start = time.time() + last_progress_time = time.time() + last_progress_val = 0 + while time.time() - poll_start < _TIMEOUT_SECONDS: + time.sleep(_POLL_INTERVAL_SECONDS) + with deps.db_update_lock: + current_status = state.get('status', 'idle') + current_progress = state.get('progress', 0) + if current_status != 'running': + break + # Stall detection — if no progress change in 10 minutes, warn. + if current_progress != last_progress_val: + last_progress_val = current_progress + last_progress_time = time.time() + elif time.time() - last_progress_time > _STALL_WARNING_SECONDS: + deps.update_progress( + automation_id, + log_line=f'{stall_label} appears stalled — waiting...', + log_type='warning', + ) + last_progress_time = time.time() # Reset so warning repeats every 10 min. + else: + # 2-hour timeout reached. + deps.update_progress( + automation_id, status='error', + phase='Timed out', log_line=timeout_label, log_type='error', + ) + return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} + + # Finished/error callback already updated the card — return matching status. + with deps.db_update_lock: + final_status = state.get('status', 'unknown') + if final_status == 'error': + return { + 'status': 'error', + 'reason': state.get('error_message', 'Unknown error'), + '_manages_own_progress': True, + } + with deps.db_update_lock: + stats = { + 'status': 'completed', '_manages_own_progress': True, + 'artists': state.get('total', 0), + 'albums': state.get('total_albums', 0), + 'tracks': state.get('total_tracks', 0), + 'removed_artists': state.get('removed_artists', 0), + 'removed_albums': state.get('removed_albums', 0), + 'removed_tracks': state.get('removed_tracks', 0), + } + stats.update(finished_extras()) + return stats diff --git a/core/automation/handlers/discover_playlist.py b/core/automation/handlers/discover_playlist.py new file mode 100644 index 00000000..90edd2a2 --- /dev/null +++ b/core/automation/handlers/discover_playlist.py @@ -0,0 +1,48 @@ +"""Automation handler: ``discover_playlist`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_discover_playlist`` closure). Kicks off background discovery +of official Spotify / iTunes metadata for mirrored playlist tracks. +The worker runs in a daemon thread and emits its own progress; this +handler returns immediately after launching it (``_manages_own_progress``). +""" + +from __future__ import annotations + +import threading +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_discover_playlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Discover official Spotify/iTunes metadata for mirrored + playlist tracks. Runs the worker in a background thread.""" + db = deps.get_database() + playlist_id = config.get('playlist_id') + discover_all = config.get('all', False) + + if discover_all: + playlists = db.get_mirrored_playlists() + elif playlist_id: + p = db.get_mirrored_playlist(int(playlist_id)) + playlists = [p] if p else [] + else: + return {'status': 'error', 'reason': 'No playlist specified'} + + if not playlists: + return {'status': 'error', 'reason': 'No playlists found'} + + threading.Thread( + target=deps.run_playlist_discovery_worker, + args=(playlists, config.get('_automation_id')), + daemon=True, + name='auto-discover-playlist', + ).start() + names = ', '.join(p['name'] for p in playlists[:3]) + return { + 'status': 'started', + 'playlist_count': str(len(playlists)), + 'playlists': names, + '_manages_own_progress': True, + } diff --git a/core/automation/handlers/download_cleanup.py b/core/automation/handlers/download_cleanup.py new file mode 100644 index 00000000..4aba711a --- /dev/null +++ b/core/automation/handlers/download_cleanup.py @@ -0,0 +1,267 @@ +"""Automation handlers: download-queue cleanup actions. + +Lifted from ``web_server._register_automation_handlers``: +- ``clean_search_history`` → :func:`auto_clean_search_history` +- ``clean_completed_downloads`` → :func:`auto_clean_completed_downloads` +- ``full_cleanup`` → :func:`auto_full_cleanup` + +All three share the download-orchestrator + tasks_lock / +download_batches / download_tasks accessors. ``full_cleanup`` is a +multi-step orchestration that pulls in quarantine purge + staging +sweep on top of the queue cleanup -- kept as one big handler since +its phases share state-detection logic. +""" + +from __future__ import annotations + +import os +import shutil as _shutil +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +# ─── clean_search_history ──────────────────────────────────────────── + + +def auto_clean_search_history(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Remove old searches from Soulseek when configured.""" + automation_id = config.get('_automation_id') + # Skip if soulseek is not the active download source or in hybrid order. + dl_mode = deps.config_manager.get('download_source.mode', 'hybrid') + hybrid_order = deps.config_manager.get( + 'download_source.hybrid_order', ['hifi', 'youtube', 'soulseek'], + ) + soulseek_active = ( + dl_mode == 'soulseek' + or (dl_mode == 'hybrid' and 'soulseek' in hybrid_order) + ) + # Reach the underlying SoulseekClient via the orchestrator's + # generic accessor. + slskd = deps.download_orchestrator.client('soulseek') if deps.download_orchestrator else None + if not soulseek_active or not slskd or not slskd.base_url: + deps.update_progress(automation_id, log_line='Soulseek not active — skipped', log_type='skip') + return {'status': 'skipped'} + if not deps.config_manager.get('soulseek.auto_clear_searches', True): + deps.update_progress( + automation_id, log_line='Auto-clear disabled in settings', log_type='skip', + ) + return {'status': 'skipped'} + try: + success = deps.run_async(deps.download_orchestrator.maintain_search_history_with_buffer( + keep_searches=50, trigger_threshold=200, + )) + if success: + deps.update_progress( + automation_id, + log_line='Search history maintenance completed', + log_type='success', + ) + return {'status': 'completed'} + else: + deps.update_progress(automation_id, log_line='No cleanup needed', log_type='skip') + return {'status': 'completed'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise + return {'status': 'error', 'error': str(e)} + + +# ─── clean_completed_downloads ─────────────────────────────────────── + + +def auto_clean_completed_downloads(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Clear completed downloads + sweep empty download directories. + Skips when active batches or post-processing is in flight.""" + automation_id = config.get('_automation_id') + try: + has_active_batches = False + has_post_processing = False + with deps.tasks_lock: + batches = deps.get_download_batches() + for batch_data in batches.values(): + if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: + has_active_batches = True + break + if not has_active_batches: + tasks = deps.get_download_tasks() + for task_data in tasks.values(): + if task_data.get('status') == 'post_processing': + has_post_processing = True + break + + if has_active_batches: + deps.update_progress( + automation_id, log_line='Skipped — downloads active', log_type='skip', + ) + return {'status': 'completed'} + + deps.run_async(deps.download_orchestrator.clear_all_completed_downloads()) + if not has_post_processing: + deps.sweep_empty_download_directories() + deps.update_progress( + automation_id, log_line='Download cleanup completed', log_type='success', + ) + return {'status': 'completed'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise + return {'status': 'error', 'reason': str(e)} + + +# ─── full_cleanup ──────────────────────────────────────────────────── + + +def auto_full_cleanup(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run all cleanup tasks: quarantine purge → download queue clear + → empty-dir sweep → staging sweep → search history.""" + automation_id = config.get('_automation_id') + steps = [] + + # --- 1. Clear quarantine --- + deps.update_progress(automation_id, phase='Clearing quarantine...', progress=0) + quarantine_path = os.path.join( + deps.docker_resolve_path(deps.config_manager.get('soulseek.download_path', './downloads')), + 'ss_quarantine', + ) + q_removed = 0 + if os.path.exists(quarantine_path): + for f in os.listdir(quarantine_path): + fp = os.path.join(quarantine_path, f) + try: + if os.path.isfile(fp): + os.remove(fp) + q_removed += 1 + elif os.path.isdir(fp): + _shutil.rmtree(fp) + q_removed += 1 + except Exception as e: # noqa: BLE001 — best-effort purge + deps.logger.debug("quarantine entry purge failed: %s", e) + steps.append(f'Quarantine: removed {q_removed} items') + deps.update_progress( + automation_id, + log_line=f'Quarantine: removed {q_removed} items', + log_type='success' if q_removed else 'info', + ) + + # --- 2. Clear completed/errored/cancelled downloads from Soulseek queue --- + deps.update_progress(automation_id, phase='Clearing download queue...', progress=20) + has_active_batches = False + has_post_processing = False + with deps.tasks_lock: + batches = deps.get_download_batches() + for batch_data in batches.values(): + if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: + has_active_batches = True + break + if not has_active_batches: + tasks = deps.get_download_tasks() + for task_data in tasks.values(): + if task_data.get('status') == 'post_processing': + has_post_processing = True + break + if has_active_batches: + steps.append('Download queue: skipped (active batches)') + deps.update_progress( + automation_id, + log_line='Download queue: skipped (active batches)', + log_type='skip', + ) + else: + try: + deps.run_async(deps.download_orchestrator.clear_all_completed_downloads()) + steps.append('Download queue: cleared') + deps.update_progress( + automation_id, log_line='Download queue: cleared', log_type='success', + ) + except Exception as e: # noqa: BLE001 — per-step best-effort + steps.append(f'Download queue: error ({e})') + deps.update_progress( + automation_id, + log_line=f'Download queue: error ({e})', + log_type='error', + ) + + # --- 3. Sweep empty download directories --- + deps.update_progress(automation_id, phase='Sweeping empty directories...', progress=40) + if has_active_batches or has_post_processing: + reason = 'active batches' if has_active_batches else 'post-processing active' + steps.append(f'Empty directories: skipped ({reason})') + deps.update_progress( + automation_id, + log_line=f'Empty directories: skipped ({reason})', + log_type='skip', + ) + else: + dirs_removed = deps.sweep_empty_download_directories() + steps.append(f'Empty directories: removed {dirs_removed}') + deps.update_progress( + automation_id, + log_line=f'Empty directories: removed {dirs_removed}', + log_type='success' if dirs_removed else 'info', + ) + + # --- 4. Sweep empty staging directories --- + deps.update_progress(automation_id, phase='Sweeping import folder...', progress=60) + staging_path = deps.get_staging_path() + s_removed = 0 + if os.path.isdir(staging_path): + for dirpath, _dirnames, _filenames in os.walk(staging_path, topdown=False): + if os.path.normpath(dirpath) == os.path.normpath(staging_path): + continue + try: + entries = os.listdir(dirpath) + except OSError: + continue + visible = [e for e in entries if not e.startswith('.')] + if not visible: + for hidden in entries: + try: + os.remove(os.path.join(dirpath, hidden)) + except Exception as e: # noqa: BLE001 — best-effort + deps.logger.debug("hidden file cleanup failed: %s", e) + try: + os.rmdir(dirpath) + s_removed += 1 + except OSError: + pass + steps.append(f'Staging: removed {s_removed} empty directories') + deps.update_progress( + automation_id, + log_line=f'Staging: removed {s_removed} empty directories', + log_type='success' if s_removed else 'info', + ) + + # --- 5. Clean search history (if enabled) --- + deps.update_progress(automation_id, phase='Cleaning search history...', progress=80) + try: + if not deps.config_manager.get('soulseek.auto_clear_searches', True): + steps.append('Search cleanup: disabled in settings') + deps.update_progress( + automation_id, log_line='Search cleanup: disabled in settings', log_type='skip', + ) + else: + deps.run_async(deps.download_orchestrator.maintain_search_history_with_buffer( + keep_searches=50, trigger_threshold=200, + )) + steps.append('Search history: cleaned') + deps.update_progress( + automation_id, log_line='Search history: cleaned', log_type='success', + ) + except Exception as e: # noqa: BLE001 — per-step best-effort + steps.append(f'Search history: error ({e})') + deps.update_progress( + automation_id, log_line=f'Search history: error ({e})', log_type='error', + ) + + total_removed = q_removed + s_removed + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', + log_line=f'Full cleanup complete — {total_removed} items removed', + log_type='success', + ) + return { + 'status': 'completed', + 'quarantine_removed': str(q_removed), + 'staging_removed': str(s_removed), + 'total_removed': str(total_removed), + 'steps': steps, + '_manages_own_progress': True, + } diff --git a/core/automation/handlers/duplicate_cleaner.py b/core/automation/handlers/duplicate_cleaner.py new file mode 100644 index 00000000..a0531622 --- /dev/null +++ b/core/automation/handlers/duplicate_cleaner.py @@ -0,0 +1,87 @@ +"""Automation handler: ``run_duplicate_cleaner`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_run_duplicate_cleaner`` closure). Submits the duplicate +cleaner to its executor, then polls the shared state dict until +the worker transitions away from ``running``. +""" + +from __future__ import annotations + +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +_TIMEOUT_SECONDS = 7200 # 2 hours +_POLL_INTERVAL_SECONDS = 3 +_INITIAL_DELAY_SECONDS = 1 + + +def auto_run_duplicate_cleaner(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Kick off the duplicate cleaner and report final stats.""" + automation_id = config.get('_automation_id') + state = deps.get_duplicate_cleaner_state() + if state.get('status') == 'running': + return {'status': 'skipped', 'reason': 'Duplicate cleaner already running'} + + # Pre-set status before submit so the polling loop doesn't see a + # stale 'finished' from a previous run. + with deps.duplicate_cleaner_lock: + state['status'] = 'running' + deps.duplicate_cleaner_executor.submit(deps.run_duplicate_cleaner) + deps.update_progress(automation_id, log_line='Duplicate cleaner started', log_type='info') + + # Monitor progress (max 2 hours). + time.sleep(_INITIAL_DELAY_SECONDS) + poll_start = time.time() + while time.time() - poll_start < _TIMEOUT_SECONDS: + time.sleep(_POLL_INTERVAL_SECONDS) + current_status = state.get('status', 'idle') + if current_status not in ('running',): + break + deps.update_progress( + automation_id, + phase=state.get('phase', 'Scanning...'), + progress=state.get('progress', 0), + processed=state.get('files_scanned', 0), + total=state.get('total_files', 0), + ) + else: + # 2-hour timeout reached. + deps.update_progress( + automation_id, status='error', + phase='Timed out', + log_line='Duplicate cleaner timed out after 2 hours', + log_type='error', + ) + return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} + + # Check actual exit status (could be 'finished' or 'error'). + final_status = state.get('status', 'idle') + if final_status == 'error': + err = state.get('error_message', 'Unknown error') + deps.update_progress( + automation_id, status='error', progress=100, + phase='Error', log_line=err, log_type='error', + ) + return {'status': 'error', 'reason': err, '_manages_own_progress': True} + + dupes = state.get('duplicates_found', 0) + removed = state.get('deleted', 0) + space_freed = state.get('space_freed', 0) + scanned = state.get('files_scanned', 0) + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', + log_line=f'Found {dupes} duplicates, removed {removed} files', + log_type='success', + ) + return { + 'status': 'completed', '_manages_own_progress': True, + 'files_scanned': scanned, + 'duplicates_found': dupes, + 'files_deleted': removed, + 'space_freed_mb': round(space_freed / (1024 * 1024), 1), + } diff --git a/core/automation/handlers/maintenance.py b/core/automation/handlers/maintenance.py new file mode 100644 index 00000000..dacd13c8 --- /dev/null +++ b/core/automation/handlers/maintenance.py @@ -0,0 +1,213 @@ +"""Automation handlers: short maintenance actions. + +Lifted from ``web_server._register_automation_handlers``: +- ``clear_quarantine`` → :func:`auto_clear_quarantine` +- ``cleanup_wishlist`` → :func:`auto_cleanup_wishlist` +- ``update_discovery_pool`` → :func:`auto_update_discovery_pool` +- ``backup_database`` → :func:`auto_backup_database` +- ``refresh_beatport_cache`` → :func:`auto_refresh_beatport_cache` + +Each is a thin wrapper around an existing service / helper. Grouped +in one module because every body is short and they share no state +between them — splitting into per-handler files would just add +import noise. +""" + +from __future__ import annotations + +import glob as _glob +import os +import shutil as _shutil +import sqlite3 +import time +from datetime import datetime +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +# ─── clear_quarantine ──────────────────────────────────────────────── + + +def auto_clear_quarantine(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Purge every file/folder under the configured ss_quarantine path.""" + automation_id = config.get('_automation_id') + quarantine_path = os.path.join( + deps.docker_resolve_path(deps.config_manager.get('soulseek.download_path', './downloads')), + 'ss_quarantine', + ) + if not os.path.exists(quarantine_path): + deps.update_progress(automation_id, log_line='No quarantine folder found', log_type='info') + return {'status': 'completed', 'removed': '0'} + removed = 0 + for f in os.listdir(quarantine_path): + fp = os.path.join(quarantine_path, f) + try: + if os.path.isfile(fp): + os.remove(fp) + removed += 1 + elif os.path.isdir(fp): + _shutil.rmtree(fp) + removed += 1 + except Exception as e: # noqa: BLE001 — best-effort purge + deps.logger.debug("quarantine entry purge failed: %s", e) + deps.update_progress( + automation_id, + log_line=f'Removed {removed} quarantined items', + log_type='success' if removed > 0 else 'info', + ) + return {'status': 'completed', 'removed': str(removed)} + + +# ─── cleanup_wishlist ──────────────────────────────────────────────── + + +def auto_cleanup_wishlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Drop duplicate entries from the wishlist for the active profile.""" + automation_id = config.get('_automation_id') + db = deps.get_database() + removed = db.remove_wishlist_duplicates(deps.get_current_profile_id()) + deps.update_progress( + automation_id, + log_line=f'Removed {removed or 0} duplicate wishlist entries', + log_type='success' if removed else 'info', + ) + return {'status': 'completed', 'removed': str(removed or 0)} + + +# ─── update_discovery_pool ─────────────────────────────────────────── + + +def auto_update_discovery_pool(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run an incremental refresh of the discovery pool via the + watchlist scanner.""" + automation_id = config.get('_automation_id') + try: + scanner = deps.get_watchlist_scanner(deps.spotify_client) + deps.update_progress(automation_id, log_line='Updating discovery pool...', log_type='info') + scanner.update_discovery_pool_incremental(deps.get_current_profile_id()) + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', log_line='Discovery pool updated', log_type='success', + ) + return {'status': 'completed', '_manages_own_progress': True} + except Exception as e: # noqa: BLE001 — automation handlers must never raise + deps.update_progress( + automation_id, status='error', + phase='Error', log_line=str(e), log_type='error', + ) + return {'status': 'error', 'reason': str(e), '_manages_own_progress': True} + + +# ─── backup_database ───────────────────────────────────────────────── + + +_MAX_BACKUPS = 5 + + +def auto_backup_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Create a hot SQLite backup, then prune old backups so only the + newest ``_MAX_BACKUPS`` remain.""" + automation_id = config.get('_automation_id') + db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') + if not os.path.exists(db_path): + return {'status': 'error', 'reason': 'Database file not found'} + + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + backup_path = f"{db_path}.backup_{timestamp}" + # Use SQLite backup API for a safe hot-copy of an active database. + src = sqlite3.connect(db_path) + dst = sqlite3.connect(backup_path) + src.backup(dst) + dst.close() + src.close() + size_mb = round(os.path.getsize(backup_path) / (1024 * 1024), 1) + + # Rolling cleanup — keep only the newest N backups. + existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime) + while len(existing) > _MAX_BACKUPS: + try: + os.remove(existing.pop(0)) + except Exception as e: # noqa: BLE001 — best-effort cleanup + deps.logger.debug("rolling backup cleanup failed: %s", e) + deps.update_progress( + automation_id, + log_line=f'Backup created: {size_mb}MB ({os.path.basename(backup_path)})', + log_type='success', + ) + return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)} + + +# ─── refresh_beatport_cache ────────────────────────────────────────── + + +_BEATPORT_SECTIONS = ( + ('hero_tracks', '/api/beatport/hero-tracks', 'Hero Tracks'), + ('new_releases', '/api/beatport/new-releases', 'New Releases'), + ('featured_charts', '/api/beatport/featured-charts', 'Featured Charts'), + ('dj_charts', '/api/beatport/dj-charts', 'DJ Charts'), + ('top_10_lists', '/api/beatport/homepage/top-10-lists', 'Top 10 Lists'), + ('top_10_releases', '/api/beatport/homepage/top-10-releases-cards', 'Top 10 Releases'), + ('hype_picks', '/api/beatport/hype-picks', 'Hype Picks'), +) + + +def auto_refresh_beatport_cache(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Refresh Beatport homepage cache by calling each endpoint internally + via Flask's ``test_client``. Invalidates the homepage cache first + so endpoints re-scrape rather than returning stale data.""" + automation_id = config.get('_automation_id') + cache = deps.get_beatport_data_cache() + # Invalidate all homepage cache timestamps so endpoints re-scrape. + with cache['cache_lock']: + for key in cache['homepage']: + cache['homepage'][key]['timestamp'] = 0 + cache['homepage'][key]['data'] = None + + refreshed = 0 + errors = [] + app = deps.get_app() + with app.test_client() as client: + for idx, (_, endpoint, label) in enumerate(_BEATPORT_SECTIONS): + deps.update_progress( + automation_id, + progress=(idx / len(_BEATPORT_SECTIONS)) * 100, + phase=f'Scraping: {label}', + current_item=label, + ) + try: + resp = client.get(endpoint) + if resp.status_code == 200: + refreshed += 1 + deps.update_progress( + automation_id, log_line=f'{label}: cached', log_type='success', + ) + else: + errors.append(label) + deps.update_progress( + automation_id, + log_line=f'{label}: HTTP {resp.status_code}', + log_type='error', + ) + except Exception as e: # noqa: BLE001 — per-section best-effort + errors.append(label) + deps.update_progress( + automation_id, + log_line=f'{label}: {str(e)}', + log_type='error', + ) + if idx < len(_BEATPORT_SECTIONS) - 1: + time.sleep(2) + + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', + log_line=f'Refreshed {refreshed}/{len(_BEATPORT_SECTIONS)} sections', + log_type='success', + ) + return { + 'status': 'completed', + 'refreshed': str(refreshed), + 'errors': str(len(errors)), + '_manages_own_progress': True, + } diff --git a/core/automation/handlers/personalized_pipeline.py b/core/automation/handlers/personalized_pipeline.py new file mode 100644 index 00000000..aa2f9190 --- /dev/null +++ b/core/automation/handlers/personalized_pipeline.py @@ -0,0 +1,356 @@ +"""Personalized Playlist Pipeline automation handler. + +Sibling to ``auto_playlist_pipeline`` (mirrored). Where the mirrored +pipeline runs REFRESH external sources → DISCOVER metadata → SYNC → +WISHLIST, the personalized pipeline is simpler: + + SNAPSHOT → SYNC → WISHLIST + +SNAPSHOT reads the persisted track list from +``PersonalizedPlaylistManager``. When ``refresh_first=True`` (config), +each playlist is refreshed BEFORE syncing — useful when the user +wants the cron to capture a fresh-each-run view (e.g. "give me a new +Hidden Gems set every night"). Default is to sync the existing +snapshot, on the assumption the user / a separate cron has already +refreshed when they wanted to. + +Config schema: + { + 'kinds': [ + {'kind': 'hidden_gems'}, + {'kind': 'time_machine', 'variant': '1980s'}, + {'kind': 'seasonal_mix', 'variant': 'halloween'}, + ... + ], + 'refresh_first': bool, # default false + 'skip_wishlist': bool, # default false + } + +Each kind dict has at minimum ``kind``; ``variant`` is required for +kinds that need it (time_machine, genre_playlist, daily_mix, +seasonal_mix). Singleton kinds (hidden_gems, discovery_shuffle, +popular_picks, fresh_tape, archives) ignore variant. + +Pipeline-running flag (``deps.state.pipeline_running``) is shared +with the mirrored pipeline so the two can't overlap. (One sync +queue, one wishlist worker — overlapping triggers would step on +each other.)""" + +from __future__ import annotations + +import json +import threading +import time +from typing import Any, Dict, List, Optional + +from core.automation.deps import AutomationDeps +from core.automation.handlers._pipeline_shared import run_sync_and_wishlist + + +# Sync state key prefix so personalized syncs don't collide with +# mirrored ones (`auto_mirror_`). +_SYNC_ID_PREFIX = 'auto_personalized' + + +def auto_personalized_pipeline(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run SNAPSHOT → SYNC → WISHLIST for selected personalized playlists.""" + deps.state.set_pipeline_running(True) + automation_id = config.get('_automation_id') + pipeline_start = time.time() + + try: + kinds_config = config.get('kinds') or [] + if not isinstance(kinds_config, list) or not kinds_config: + deps.state.set_pipeline_running(False) + return { + 'status': 'error', + 'error': 'No personalized playlist kinds selected', + } + + refresh_first = bool(config.get('refresh_first', False)) + skip_wishlist = bool(config.get('skip_wishlist', False)) + + manager = deps.build_personalized_manager() + + deps.update_progress( + automation_id, + progress=2, + phase=f'Personalized pipeline: {len(kinds_config)} playlist(s)', + log_line=f'Starting pipeline for {len(kinds_config)} playlist(s)', + log_type='info', + ) + + # ── PHASE 1: SNAPSHOT (optionally refresh) ────────────────── + deps.update_progress( + automation_id, + progress=3, + phase='Phase 1/2: Loading snapshots...' if not refresh_first + else 'Phase 1/2: Refreshing snapshots...', + log_line='Phase 1: Snapshot' + (' (with refresh)' if refresh_first else ''), + log_type='info', + ) + + profile_id = deps.get_current_profile_id() + playload_payloads = _build_payloads_for_kinds( + deps, manager, kinds_config, profile_id, + automation_id=automation_id, + refresh_first=refresh_first, + ) + + if not playload_payloads: + deps.state.set_pipeline_running(False) + deps.update_progress( + automation_id, + status='finished', progress=100, + phase='No playlists to sync', + log_line='No personalized playlists had tracks to sync', + log_type='warning', + ) + return { + 'status': 'completed', + '_manages_own_progress': True, + 'playlists_synced': '0', + 'tracks_synced': '0', + 'duration_seconds': str(int(time.time() - pipeline_start)), + } + + deps.update_progress( + automation_id, + progress=50, + phase='Phase 1/2: Snapshot complete', + log_line=f'Phase 1 done: {len(playload_payloads)} playlist(s) ready to sync', + log_type='success', + ) + + # ── PHASE 2: SYNC + WISHLIST (shared helper) ──────────────── + sync_summary = run_sync_and_wishlist( + deps, + automation_id, + playload_payloads, + sync_one_fn=lambda pl: _sync_personalized_playlist(deps, pl), + sync_id_for_fn=lambda pl: pl['sync_id'], + skip_wishlist=skip_wishlist, + progress_start=51, + progress_end=90, + sync_phase_label='Phase 2/2: Syncing to server...', + sync_phase_start_log='Phase 2: Sync', + wishlist_phase_label='Phase 2/2: Processing wishlist...', + wishlist_phase_start_log='Wishlist', + ) + + # ── COMPLETE ──────────────────────────────────────────────── + duration = int(time.time() - pipeline_start) + deps.update_progress( + automation_id, + status='finished', progress=100, + phase='Pipeline complete', + log_line=f'Personalized pipeline finished in {duration // 60}m {duration % 60}s', + log_type='success', + ) + + deps.state.set_pipeline_running(False) + return { + 'status': 'completed', + '_manages_own_progress': True, + 'playlists_synced': str(len(playload_payloads)), + 'tracks_synced': str(sync_summary['synced']), + 'sync_skipped': str(sync_summary['skipped']), + 'wishlist_queued': str(sync_summary['wishlist_queued']), + 'duration_seconds': str(duration), + } + + except Exception as e: # noqa: BLE001 — automation handlers must never raise into engine + deps.state.set_pipeline_running(False) + deps.update_progress( + automation_id, + status='error', progress=100, + phase='Pipeline error', + log_line=f'Personalized pipeline failed: {e}', + log_type='error', + ) + return {'status': 'error', 'error': str(e), '_manages_own_progress': True} + + +def _build_payloads_for_kinds( + deps: AutomationDeps, + manager: Any, + kinds_config: List[Dict[str, Any]], + profile_id: int, + *, + automation_id: Optional[str], + refresh_first: bool, +) -> List[Dict[str, Any]]: + """Resolve each requested kind+variant into a sync-payload dict. + + Each payload has: ``{'name', 'kind', 'variant', 'tracks_json', + 'image_url', 'sync_id'}``. Playlists with no tracks (e.g. a + seasonal mix that hasn't been populated yet) are omitted from + the result so the sync loop doesn't waste time on empty pushes. + """ + payloads: List[Dict[str, Any]] = [] + for entry in kinds_config: + if not isinstance(entry, dict): + continue + kind = entry.get('kind') + variant = entry.get('variant') or '' + if not kind: + continue + + try: + # Refresh when ANY of: + # - explicit user flag (cron use case: regenerate each run) + # - snapshot marked stale by upstream data refresher + # - playlist was never generated yet (auto-created by + # ensure_playlist; track_count=0, last_generated_at=NULL). + # Without this branch, a first-run pipeline reads the + # empty snapshot and silently skips — user picks a kind, + # hits run, gets "No tracks to sync" with no clue why. + if refresh_first: + record = manager.refresh_playlist(kind, variant, profile_id) + else: + existing = manager.ensure_playlist(kind, variant, profile_id) + needs_first_gen = existing.last_generated_at is None + if existing.is_stale or needs_first_gen: + record = manager.refresh_playlist(kind, variant, profile_id) + else: + record = existing + except Exception as exc: # noqa: BLE001 — log + continue with next kind + deps.update_progress( + automation_id, + log_line=f'Skipping {kind}{("/" + variant) if variant else ""}: {exc}', + log_type='warning', + ) + continue + + tracks = manager.get_playlist_tracks(record.id) + if not tracks: + deps.update_progress( + automation_id, + log_line=f'No tracks in {record.name} — skipping sync', + log_type='skip', + ) + continue + + tracks_json = [_track_to_sync_shape(t) for t in tracks] + payloads.append({ + 'name': record.name, + 'kind': record.kind, + 'variant': record.variant, + 'tracks_json': tracks_json, + 'image_url': '', # personalized playlists don't have a cover image yet + 'sync_id': f'{_SYNC_ID_PREFIX}_{record.kind}_{record.variant or "_"}', + }) + return payloads + + +def _track_to_sync_shape(track: Any) -> Dict[str, Any]: + """Convert a personalized.types.Track into the dict shape + `_run_sync_task` expects. Mirrors what the mirrored pipeline + builds from extra_data.matched_data, preserving enriched metadata + from personalized snapshots when available.""" + primary_id = track.spotify_track_id or track.itunes_track_id or track.deezer_track_id or '' + rich_data = _coerce_track_data_json(getattr(track, 'track_data_json', None)) + + if not rich_data: + album = {'name': track.album_name or ''} + cover_url = getattr(track, 'album_cover_url', None) + if cover_url: + album['images'] = [{'url': cover_url}] + return { + 'name': track.track_name, + 'artists': [{'name': track.artist_name}], + 'album': album, + 'duration_ms': int(track.duration_ms or 0), + 'id': primary_id, + } + + payload = dict(rich_data) + cover_url = ( + getattr(track, 'album_cover_url', None) + or payload.get('album_cover_url') + or payload.get('image_url') + ) + payload['id'] = payload.get('id') or primary_id + payload['name'] = payload.get('name') or track.track_name + payload['artists'] = _normalize_artists(payload.get('artists'), track.artist_name) + payload['album'] = _normalize_album(payload.get('album'), track, cover_url=cover_url) + payload['duration_ms'] = int(payload.get('duration_ms') or track.duration_ms or 0) + if 'popularity' not in payload and getattr(track, 'popularity', None) is not None: + payload['popularity'] = int(track.popularity or 0) + + if cover_url and not payload.get('image_url'): + payload['image_url'] = cover_url + + return payload + + +def _coerce_track_data_json(value: Any) -> Dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, str) and value.strip(): + try: + loaded = json.loads(value) + except (TypeError, ValueError): + return {} + return loaded if isinstance(loaded, dict) else {} + return {} + + +def _normalize_artists(artists: Any, fallback_artist: str) -> List[Dict[str, Any]]: + if not artists: + return [{'name': fallback_artist or 'Unknown Artist'}] + if isinstance(artists, list): + normalized = [] + for artist in artists: + if isinstance(artist, dict): + normalized.append(artist if artist.get('name') else {'name': str(artist)}) + elif isinstance(artist, str): + normalized.append({'name': artist}) + else: + normalized.append({'name': str(artist)}) + return normalized or [{'name': fallback_artist or 'Unknown Artist'}] + if isinstance(artists, dict): + return [artists if artists.get('name') else {'name': str(artists)}] + return [{'name': str(artists)}] + + +def _normalize_album(album: Any, track: Any, cover_url: Optional[str] = None) -> Dict[str, Any]: + if isinstance(album, dict): + normalized = dict(album) + else: + normalized = {'name': str(album) if album else (track.album_name or '')} + + normalized['name'] = normalized.get('name') or track.album_name or '' + images = normalized.get('images') + if not isinstance(images, list): + images = [] + if cover_url and not images: + images = [{'url': cover_url}] + if images: + normalized['images'] = images + return normalized + + +def _sync_personalized_playlist(deps: AutomationDeps, payload: Dict[str, Any]) -> Dict[str, Any]: + """Launch a personalized playlist sync via _run_sync_task on a + daemon thread + return immediately with status='started'. + + Mirrors the mirrored ``auto_sync_playlist`` return contract so the + shared helper can poll on ``sync_states[sync_id]`` and aggregate + results identically.""" + sync_id = payload['sync_id'] + name = payload['name'] + tracks_json = payload['tracks_json'] + profile_id = deps.get_current_profile_id() + + threading.Thread( + target=deps.run_sync_task, + args=(sync_id, name, tracks_json, None, profile_id, payload.get('image_url', '')), + daemon=True, + name=f'auto-personalized-{sync_id}', + ).start() + return { + 'status': 'started', + 'playlist_name': name, + '_manages_own_progress': True, + } diff --git a/core/automation/handlers/playlist_pipeline.py b/core/automation/handlers/playlist_pipeline.py new file mode 100644 index 00000000..8fc98df2 --- /dev/null +++ b/core/automation/handlers/playlist_pipeline.py @@ -0,0 +1,227 @@ +"""Automation handler: ``playlist_pipeline`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_playlist_pipeline`` closure). Runs the full playlist +lifecycle in a single trigger: + + Phase 1: REFRESH -- pull fresh track lists from sources + Phase 2: DISCOVER -- look up official Spotify/iTunes metadata + Phase 3: SYNC -- push the result to the active media server + Phase 4: WISHLIST -- queue any missing tracks for download + +Each phase emits its own progress range so the trigger card shows +useful per-phase percentages instead of "loading...". Phase 4 is +optional via ``skip_wishlist`` config. + +Composition: this handler invokes ``auto_refresh_mirrored`` and +``auto_sync_playlist`` directly (passing ``_automation_id: None`` so +the sub-handlers don't hijack pipeline progress) instead of going +through the engine — keeps the four phases observable as one +trigger from the user's perspective. Pipeline-level guard +(``state.pipeline_running``) prevents overlapping runs. +""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps +from core.automation.handlers._pipeline_shared import run_sync_and_wishlist +from core.automation.handlers.refresh_mirrored import auto_refresh_mirrored +from core.automation.handlers.sync_playlist import auto_sync_playlist + + +# Per-playlist sync poll cap inside Phase 3. +# Discovery poll cap inside Phase 2. +_DISCOVERY_TIMEOUT_SECONDS = 3600 + + +def auto_playlist_pipeline(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run REFRESH → DISCOVER → SYNC → WISHLIST in sequence. + + Sets / clears ``deps.state.pipeline_running`` around the whole + run so the registration guard can short-circuit overlapping + triggers. + """ + deps.state.set_pipeline_running(True) + automation_id = config.get('_automation_id') + pipeline_start = time.time() + + try: + db = deps.get_database() + playlist_id = config.get('playlist_id') + process_all = config.get('all', False) + skip_wishlist = config.get('skip_wishlist', False) + + # Resolve playlists. + if process_all: + playlists = db.get_mirrored_playlists() + elif playlist_id: + p = db.get_mirrored_playlist(int(playlist_id)) + playlists = [p] if p else [] + else: + deps.state.set_pipeline_running(False) + return {'status': 'error', 'error': 'No playlist specified'} + + playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] + if not playlists: + deps.state.set_pipeline_running(False) + return {'status': 'error', 'error': 'No refreshable playlists found'} + + pl_names = ', '.join(p.get('name', '?') for p in playlists[:3]) + if len(playlists) > 3: + pl_names += f' (+{len(playlists) - 3} more)' + + deps.update_progress( + automation_id, + progress=2, + phase=f'Pipeline: {len(playlists)} playlist(s)', + log_line=f'Starting pipeline for: {pl_names}', + log_type='info', + ) + + # ── PHASE 1: REFRESH ────────────────────────────────────────── + deps.update_progress( + automation_id, + progress=3, + phase='Phase 1/4: Refreshing playlists...', + log_line='Phase 1: Refresh', + log_type='info', + ) + + refresh_config = dict(config) + refresh_config['_automation_id'] = None # Don't let sub-handler hijack pipeline progress. + refresh_result = auto_refresh_mirrored(refresh_config, deps) + refreshed = int(refresh_result.get('refreshed', 0)) + refresh_errors = int(refresh_result.get('errors', 0)) + + deps.update_progress( + automation_id, + progress=25, + phase='Phase 1/4: Refresh complete', + log_line=f'Phase 1 done: {refreshed} refreshed, {refresh_errors} errors', + log_type='success' if refresh_errors == 0 else 'warning', + ) + + # ── PHASE 2: DISCOVER ───────────────────────────────────────── + deps.update_progress( + automation_id, + progress=26, + phase='Phase 2/4: Discovering metadata...', + log_line='Phase 2: Discover', + log_type='info', + ) + + # Reload playlists (refresh may have updated them). + if process_all: + disc_playlists = db.get_mirrored_playlists() + else: + disc_playlists = [db.get_mirrored_playlist(int(playlist_id))] + disc_playlists = [p for p in disc_playlists if p] + + # Run discovery in a thread and wait for it. + disc_done = threading.Event() + + def _disc_wrapper(pls): + try: + # The worker updates automation_progress internally, + # but we pass None so it doesn't conflict with our + # pipeline progress. + deps.run_playlist_discovery_worker(pls, automation_id=None) + except Exception as e: + deps.logger.error(f"[Pipeline] Discovery error: {e}") + finally: + disc_done.set() + + threading.Thread( + target=_disc_wrapper, args=(disc_playlists,), + daemon=True, name='pipeline-discover', + ).start() + + # Poll for completion with progress updates. + poll_start = time.time() + while not disc_done.wait(timeout=3): + elapsed = int(time.time() - poll_start) + deps.update_progress( + automation_id, + progress=min(26 + elapsed // 4, 54), + phase=f'Phase 2/4: Discovering... ({elapsed}s)', + ) + if elapsed > _DISCOVERY_TIMEOUT_SECONDS: + deps.update_progress( + automation_id, + log_line='Discovery timed out after 1 hour', + log_type='warning', + ) + break + + deps.update_progress( + automation_id, + progress=55, + phase='Phase 2/4: Discovery complete', + log_line='Phase 2 done: discovery complete', + log_type='success', + ) + + # ── PHASE 3 + 4: SYNC + WISHLIST (delegated to shared helper) ── + # Each mirrored playlist payload only needs `id` + `name` for + # the helper; `auto_sync_playlist` reads the rest from the + # mirrored DB by id. + sync_summary = run_sync_and_wishlist( + deps, + automation_id, + [pl for pl in playlists if pl.get('id')], + sync_one_fn=lambda pl: auto_sync_playlist( + {'playlist_id': str(pl['id']), '_automation_id': None}, + deps, + ), + sync_id_for_fn=lambda pl: f"auto_mirror_{pl['id']}", + skip_wishlist=skip_wishlist, + progress_start=56, + progress_end=85, + sync_phase_label='Phase 3/4: Syncing to server...', + sync_phase_start_log='Phase 3: Sync', + wishlist_phase_label='Phase 4/4: Processing wishlist...', + wishlist_phase_start_log='Phase 4: Wishlist', + ) + total_synced = sync_summary['synced'] + total_skipped = sync_summary['skipped'] + sync_errors = sync_summary['errors'] + wishlist_queued = sync_summary['wishlist_queued'] + + # ── COMPLETE ────────────────────────────────────────────────── + duration = int(time.time() - pipeline_start) + deps.update_progress( + automation_id, + status='finished', + progress=100, + phase='Pipeline complete', + log_line=f'Pipeline finished in {duration // 60}m {duration % 60}s', + log_type='success', + ) + + deps.state.set_pipeline_running(False) + return { + 'status': 'completed', + '_manages_own_progress': True, + 'playlists_refreshed': str(refreshed), + 'tracks_discovered': 'completed', + 'tracks_synced': str(total_synced), + 'sync_skipped': str(total_skipped), + 'wishlist_queued': str(wishlist_queued), + 'duration_seconds': str(duration), + } + + except Exception as e: + deps.state.set_pipeline_running(False) + deps.update_progress( + automation_id, + status='error', + progress=100, + phase='Pipeline error', + log_line=f'Pipeline failed: {e}', + log_type='error', + ) + return {'status': 'error', 'error': str(e), '_manages_own_progress': True} diff --git a/core/automation/handlers/process_wishlist.py b/core/automation/handlers/process_wishlist.py new file mode 100644 index 00000000..27a08046 --- /dev/null +++ b/core/automation/handlers/process_wishlist.py @@ -0,0 +1,27 @@ +"""Automation handler: ``process_wishlist`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_process_wishlist`` closure). Wishlist processing is async — +the helper submits a batch to an executor and returns immediately; +per-track stats arrive later via batch-completion callbacks. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_process_wishlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Kick off the wishlist processor for an automation trigger. + + Returns immediately after submission; the wishlist worker emits + per-batch progress via its own callbacks. We only report + ``status: completed`` to mark the trigger fired successfully. + """ + try: + deps.process_wishlist_automatically(automation_id=config.get('_automation_id')) + return {'status': 'completed'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise into the engine + return {'status': 'error', 'error': str(e)} diff --git a/core/automation/handlers/progress_callbacks.py b/core/automation/handlers/progress_callbacks.py new file mode 100644 index 00000000..a097864b --- /dev/null +++ b/core/automation/handlers/progress_callbacks.py @@ -0,0 +1,89 @@ +"""Progress + history callbacks the automation engine invokes around +each handler run. + +Lifted from the closures at the bottom of +``web_server._register_automation_handlers``: +- ``_progress_init`` → :func:`progress_init` +- ``_progress_finish`` → :func:`progress_finish` +- ``_record_automation_history`` → :func:`record_history` +- ``_on_library_scan_completed`` → :func:`on_library_scan_completed` + +The engine accepts four callables via +``register_progress_callbacks(init, finish, update, history)``; +``registration.register_all`` wires these here. The +``library_scan_completed`` callback is registered separately on the +``web_scan_manager`` (when one is available) -- see +``register_library_scan_completed_emitter``. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def progress_init(aid: Any, name: str, action_type: str, deps: AutomationDeps) -> None: + """Initialize per-automation progress state when the engine starts + a handler. Thin wrapper so the engine receives a closure that + delegates into the live progress tracker.""" + deps.init_automation_progress(aid, name, action_type) + + +def progress_finish(aid: Any, result: Dict[str, Any], deps: AutomationDeps) -> None: + """Emit the final progress update when a handler returns. + + Skipped for handlers that manage their own progress lifecycle + (they call ``update_progress(status='finished')`` themselves and + set ``_manages_own_progress: True`` in the returned dict). + Otherwise translates the handler's status into a finished/error + progress emit with a status-appropriate phase + log line. + """ + if result.get('_manages_own_progress'): + return + result_status = result.get('status', '') + status = 'error' if result_status == 'error' else 'finished' + msg = result.get('error', result.get('reason', result_status or 'done')) + deps.update_progress( + aid, + status=status, + progress=100, + phase='Error' if status == 'error' else 'Complete', + log_line=msg, + log_type='error' if status == 'error' else 'success', + ) + + +def record_history(aid: Any, result: Dict[str, Any], deps: AutomationDeps) -> None: + """Capture progress state into run history before the engine's + cleanup pass clears it. Thin wrapper so the engine sees a stable + callable.""" + deps.record_progress_history(aid, result, deps.get_database()) + + +def on_library_scan_completed(deps: AutomationDeps) -> None: + """Emit the ``library_scan_completed`` automation event with the + active media-server type. Replaces the hard-coded + ``scan_completion_callback → trigger_automatic_database_update`` + chain so any automation can listen for scan completion as a + trigger.""" + if not deps.engine: + return + server_type = ( + getattr(deps.web_scan_manager, '_current_server_type', None) + or 'unknown' + ) + deps.engine.emit('library_scan_completed', { + 'server_type': server_type, + }) + + +def register_library_scan_completed_emitter(deps: AutomationDeps) -> None: + """Wire :func:`on_library_scan_completed` to the + ``web_scan_manager``'s scan-completion callback list. No-op when + no scan manager is configured (e.g. headless / test contexts).""" + if not deps.web_scan_manager: + return + deps.web_scan_manager.add_scan_completion_callback( + lambda: on_library_scan_completed(deps), + ) diff --git a/core/automation/handlers/quality_scanner.py b/core/automation/handlers/quality_scanner.py new file mode 100644 index 00000000..69ec3f02 --- /dev/null +++ b/core/automation/handlers/quality_scanner.py @@ -0,0 +1,83 @@ +"""Automation handler: ``start_quality_scan`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_start_quality_scan`` closure). Submits the quality scanner +to its executor with the configured scope (default: ``watchlist``) +then polls the shared state dict. +""" + +from __future__ import annotations + +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +_TIMEOUT_SECONDS = 7200 # 2 hours +_POLL_INTERVAL_SECONDS = 3 +_INITIAL_DELAY_SECONDS = 1 + + +def auto_start_quality_scan(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + automation_id = config.get('_automation_id') + state = deps.get_quality_scanner_state() + if state.get('status') == 'running': + return {'status': 'skipped', 'reason': 'Quality scan already running'} + + scope = config.get('scope', 'watchlist') + # Pre-set status before submit so the polling loop doesn't see a + # stale 'finished' from a previous run. + with deps.quality_scanner_lock: + state['status'] = 'running' + deps.quality_scanner_executor.submit(deps.run_quality_scanner, scope, deps.get_current_profile_id()) + deps.update_progress( + automation_id, log_line=f'Quality scan started (scope: {scope})', log_type='info', + ) + + # Monitor progress (max 2 hours). + time.sleep(_INITIAL_DELAY_SECONDS) + poll_start = time.time() + while time.time() - poll_start < _TIMEOUT_SECONDS: + time.sleep(_POLL_INTERVAL_SECONDS) + current_status = state.get('status', 'idle') + if current_status not in ('running',): + break + deps.update_progress( + automation_id, + phase=state.get('phase', 'Scanning...'), + progress=state.get('progress', 0), + processed=state.get('processed', 0), + total=state.get('total', 0), + ) + else: + deps.update_progress( + automation_id, status='error', + phase='Timed out', log_line='Quality scan timed out after 2 hours', + log_type='error', + ) + return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} + + final_status = state.get('status', 'idle') + if final_status == 'error': + err = state.get('error_message', 'Unknown error') + deps.update_progress( + automation_id, status='error', progress=100, + phase='Error', log_line=err, log_type='error', + ) + return {'status': 'error', 'reason': err, '_manages_own_progress': True} + + issues = state.get('low_quality', 0) + deps.update_progress( + automation_id, status='finished', progress=100, + phase='Complete', + log_line=f'Quality scan complete — {issues} issues found', + log_type='success', + ) + return { + 'status': 'completed', 'scope': scope, '_manages_own_progress': True, + 'tracks_scanned': state.get('processed', 0), + 'quality_met': state.get('quality_met', 0), + 'low_quality': issues, + 'matched': state.get('matched', 0), + } diff --git a/core/automation/handlers/refresh_mirrored.py b/core/automation/handlers/refresh_mirrored.py new file mode 100644 index 00000000..76ea32ce --- /dev/null +++ b/core/automation/handlers/refresh_mirrored.py @@ -0,0 +1,308 @@ +"""Automation handler: ``refresh_mirrored`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_refresh_mirrored`` closure). Re-pulls track lists from each +mirrored playlist's source (Spotify / Tidal / Deezer / YouTube), +updates the local mirror DB, and emits a ``playlist_changed`` +automation event when the track set actually shifts. + +Source-specific branches (Spotify auth + public-embed fallback, +``spotify_public`` URL→ID resolution, Deezer / Tidal / YouTube) +remain identical to the pre-extraction closure — this is a +mechanical lift, not a redesign. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_refresh_mirrored(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Refresh mirrored playlist(s) from source. + + Returns ``{'status': 'completed', 'refreshed': '', + 'errors': ''}`` on success (counts stringified to match the + automation engine's stat-rendering convention). + """ + db = deps.get_database() + playlist_id = config.get('playlist_id') + refresh_all = config.get('all', False) + auto_id = config.get('_automation_id') + + if refresh_all: + playlists = db.get_mirrored_playlists() + elif playlist_id: + p = db.get_mirrored_playlist(int(playlist_id)) + playlists = [p] if p else [] + else: + return {'status': 'error', 'reason': 'No playlist specified'} + + # Filter out sources that can't be refreshed (no external API). + playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] + + refreshed = 0 + errors = [] + for idx, pl in enumerate(playlists): + try: + source = pl.get('source', '') + source_id = pl.get('source_playlist_id', '') + deps.update_progress( + auto_id, + progress=(idx / max(1, len(playlists))) * 100, + phase=f'Refreshing: "{pl.get("name", "")}"', + current_item=pl.get('name', ''), + ) + tracks = None + + if source == 'spotify': + # Try authenticated API first, fall back to public embed scraper. + if deps.spotify_client and deps.spotify_client.is_spotify_authenticated(): + playlist_obj = deps.spotify_client.get_playlist_by_id(source_id) + if playlist_obj and playlist_obj.tracks: + tracks = [] + for t in playlist_obj.tracks: + artist_name = t.artists[0] if t.artists else '' + track_dict = { + 'track_name': t.name or '', + 'artist_name': str(artist_name), + 'album_name': t.album or '', + 'duration_ms': t.duration_ms or 0, + 'source_track_id': t.id or '', + } + # Spotify data IS official — auto-mark as discovered. + if t.id: + _album_obj = {'name': t.album or ''} + if getattr(t, 'image_url', None): + _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] + track_dict['extra_data'] = json.dumps({ + 'discovered': True, + 'provider': 'spotify', + 'confidence': 1.0, + 'matched_data': { + 'id': t.id, + 'name': t.name or '', + 'artists': [{'name': str(a)} for a in (t.artists or [])], + 'album': _album_obj, + 'duration_ms': t.duration_ms or 0, + 'image_url': getattr(t, 'image_url', None), + } + }) + tracks.append(track_dict) + + # Fallback: public embed scraper (no auth needed). + if tracks is None: + try: + from core.spotify_public_scraper import scrape_spotify_embed + embed_data = scrape_spotify_embed('playlist', source_id) + if embed_data and not embed_data.get('error') and embed_data.get('tracks'): + embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' + tracks = [] + for t in embed_data['tracks']: + artist_names = [a['name'] for a in t.get('artists', [])] + artist_name = artist_names[0] if artist_names else '' + track_dict = { + 'track_name': t.get('name', ''), + 'artist_name': artist_name, + 'album_name': embed_album, + 'duration_ms': t.get('duration_ms', 0), + 'source_track_id': t.get('id', ''), + } + # Store Spotify track ID hint but don't mark discovered — + # Discover step needs to run for proper album art. + if t.get('id'): + track_dict['extra_data'] = json.dumps({ + 'discovered': False, + 'spotify_hint': { + 'id': t['id'], + 'name': t.get('name', ''), + 'artists': t.get('artists', []), + } + }) + tracks.append(track_dict) + except Exception as e: + deps.logger.warning(f"Spotify public scraper fallback failed for {source_id}: {e}") + + elif source == 'spotify_public': + # source_playlist_id is an MD5 hash; extract actual Spotify ID from stored description (URL). + try: + from core.spotify_public_scraper import parse_spotify_url, scrape_spotify_embed + spotify_url = pl.get('description', '') + parsed = parse_spotify_url(spotify_url) if spotify_url else None + + # If Spotify is authenticated, use the full API (auto-discovers with album art). + if (parsed and parsed.get('type') == 'playlist' + and deps.spotify_client and deps.spotify_client.is_spotify_authenticated()): + playlist_obj = deps.spotify_client.get_playlist_by_id(parsed['id']) + if playlist_obj and playlist_obj.tracks: + tracks = [] + for t in playlist_obj.tracks: + artist_name = t.artists[0] if t.artists else '' + track_dict = { + 'track_name': t.name or '', + 'artist_name': str(artist_name), + 'album_name': t.album or '', + 'duration_ms': t.duration_ms or 0, + 'source_track_id': t.id or '', + } + if t.id: + _album_obj = {'name': t.album or ''} + if getattr(t, 'image_url', None): + _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] + track_dict['extra_data'] = json.dumps({ + 'discovered': True, + 'provider': 'spotify', + 'confidence': 1.0, + 'matched_data': { + 'id': t.id, + 'name': t.name or '', + 'artists': [{'name': str(a)} for a in (t.artists or [])], + 'album': _album_obj, + 'duration_ms': t.duration_ms or 0, + 'image_url': getattr(t, 'image_url', None), + } + }) + tracks.append(track_dict) + + # Fallback: public embed scraper (no auth or album-type URL). + if tracks is None and parsed: + embed_data = scrape_spotify_embed(parsed['type'], parsed['id']) + if embed_data and not embed_data.get('error') and embed_data.get('tracks'): + embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' + tracks = [] + for t in embed_data['tracks']: + artist_names = [a['name'] for a in t.get('artists', [])] + artist_name = artist_names[0] if artist_names else '' + tracks.append({ + 'track_name': t.get('name', ''), + 'artist_name': artist_name, + 'album_name': embed_album, + 'duration_ms': t.get('duration_ms', 0), + 'source_track_id': t.get('id', ''), + }) + # No extra_data — let preservation code keep existing discovery data. + except Exception as e: + deps.logger.warning(f"Spotify public playlist refresh failed for {source_id}: {e}") + + elif source == 'deezer': + try: + deezer = deps.get_deezer_client() + playlist_data = deezer.get_playlist(source_id) + if playlist_data and playlist_data.get('tracks'): + tracks = [] + for t in playlist_data['tracks']: + artist_name = t['artists'][0] if t.get('artists') else '' + tracks.append({ + 'track_name': t.get('name', ''), + 'artist_name': str(artist_name), + 'album_name': t.get('album', ''), + 'duration_ms': t.get('duration_ms', 0), + 'source_track_id': str(t.get('id', '')), + }) + except Exception as e: + deps.logger.warning(f"Deezer playlist refresh failed for {source_id}: {e}") + + elif source == 'tidal': + if not deps.tidal_client or not deps.tidal_client.is_authenticated(): + deps.logger.warning(f"Tidal not authenticated — skipping refresh for '{pl.get('name', '')}'") + deps.update_progress( + auto_id, + log_line=f'Skipped "{pl.get("name", "")}" — Tidal not authenticated', + log_type='skip', + ) + continue + full_playlist = deps.tidal_client.get_playlist(source_id) + if full_playlist and full_playlist.tracks: + tracks = [] + for t in full_playlist.tracks: + artist_name = t.artists[0] if t.artists else '' + tracks.append({ + 'track_name': t.name or '', + 'artist_name': str(artist_name), + 'album_name': t.album or '', + 'duration_ms': t.duration_ms or 0, + 'source_track_id': t.id or '', + }) + + elif source == 'youtube': + # source_playlist_id is now a deterministic hash; use stored description (original URL) for refresh. + yt_url = pl.get('description', '') or f"https://www.youtube.com/playlist?list={source_id}" + playlist_data = deps.parse_youtube_playlist(yt_url) + if playlist_data and playlist_data.get('tracks'): + tracks = [] + for t in playlist_data['tracks']: + artist_name = t['artists'][0] if t.get('artists') else '' + tracks.append({ + 'track_name': t.get('name', ''), + 'artist_name': str(artist_name), + 'album_name': '', + 'duration_ms': t.get('duration_ms', 0), + 'source_track_id': t.get('id', ''), + }) + + if tracks is not None: + # Compare old vs new track IDs to detect changes. + old_tracks = db.get_mirrored_playlist_tracks(pl['id']) if pl.get('id') else [] + old_ids = {t.get('source_track_id') for t in old_tracks if t.get('source_track_id')} + new_ids = {t.get('source_track_id') for t in tracks if t.get('source_track_id')} + + # Preserve existing discovery extra_data for tracks that still exist. + old_extra_map = db.get_mirrored_tracks_extra_data_map(pl['id']) if pl.get('id') else {} + for t in tracks: + sid = t.get('source_track_id', '') + if sid and sid in old_extra_map and 'extra_data' not in t: + t['extra_data'] = old_extra_map[sid] + + db.mirror_playlist( + source=source, + source_playlist_id=source_id, + name=pl['name'], + tracks=tracks, + profile_id=pl.get('profile_id', 1), + owner=pl.get('owner'), + image_url=pl.get('image_url'), + ) + refreshed += 1 + + # Emit playlist_changed if tracks actually changed. + if old_ids != new_ids: + added_count = len(new_ids - old_ids) + removed_count = len(old_ids - new_ids) + deps.logger.info( + f"[AUTOMATION] Playlist changed: '{pl.get('name', '')}' — " + f"{added_count} added, {removed_count} removed (old={len(old_ids)}, new={len(new_ids)})" + ) + deps.update_progress( + auto_id, + log_line=f'"{pl.get("name", "")}" — {added_count} added, {removed_count} removed', + log_type='success', + ) + try: + if deps.engine: + deps.engine.emit('playlist_changed', { + 'playlist_name': pl.get('name', ''), + 'playlist_id': str(pl.get('id', '')), + 'old_count': str(len(old_ids)), + 'new_count': str(len(new_ids)), + 'added': str(added_count), + 'removed': str(removed_count), + }) + except Exception as e: + deps.logger.debug("playlist_synced automation emit failed: %s", e) + else: + deps.logger.warning(f"[AUTOMATION] No changes: '{pl.get('name', '')}' (tracks={len(old_ids)})") + deps.update_progress( + auto_id, + log_line=f'No changes: "{pl.get("name", "")}"', + log_type='skip', + ) + except Exception as e: + errors.append(f"{pl.get('name', '?')}: {str(e)}") + deps.update_progress( + auto_id, + log_line=f'Error: {pl.get("name", "?")} — {str(e)}', + log_type='error', + ) + return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors))} diff --git a/core/automation/handlers/registration.py b/core/automation/handlers/registration.py new file mode 100644 index 00000000..94f23db6 --- /dev/null +++ b/core/automation/handlers/registration.py @@ -0,0 +1,184 @@ +"""One-stop registration of every extracted automation handler. + +``web_server`` builds the deps once at startup and calls +:func:`register_all` here. Each new handler module gets one line in +this file when it lands. +""" + +from __future__ import annotations + +from core.automation.deps import AutomationDeps +from core.automation.handlers.process_wishlist import auto_process_wishlist +from core.automation.handlers.scan_watchlist import auto_scan_watchlist +from core.automation.handlers.scan_library import auto_scan_library +from core.automation.handlers.refresh_mirrored import auto_refresh_mirrored +from core.automation.handlers.sync_playlist import auto_sync_playlist +from core.automation.handlers.discover_playlist import auto_discover_playlist +from core.automation.handlers.playlist_pipeline import auto_playlist_pipeline +from core.automation.handlers.personalized_pipeline import auto_personalized_pipeline +from core.automation.handlers.database_update import ( + auto_start_database_update, auto_deep_scan_library, +) +from core.automation.handlers.duplicate_cleaner import auto_run_duplicate_cleaner +from core.automation.handlers.quality_scanner import auto_start_quality_scan +from core.automation.handlers.maintenance import ( + auto_clear_quarantine, + auto_cleanup_wishlist, + auto_update_discovery_pool, + auto_backup_database, + auto_refresh_beatport_cache, +) +from core.automation.handlers.download_cleanup import ( + auto_clean_search_history, + auto_clean_completed_downloads, + auto_full_cleanup, +) +from core.automation.handlers.run_script import auto_run_script +from core.automation.handlers.search_and_download import auto_search_and_download +from core.automation.handlers.progress_callbacks import ( + progress_init, + progress_finish, + record_history, + register_library_scan_completed_emitter, +) + + +def register_all(deps: AutomationDeps) -> None: + """Wire every extracted handler to the engine. + + Each ``register_action_handler`` call binds the action name (the + string the trigger uses to look up its action) to a thin lambda + that injects ``deps`` and forwards the engine-supplied config. + Guards stay alongside their handler so duplicate-run prevention + behaves identically to the pre-extraction code. + """ + engine = deps.engine + + # Self-guards prevent duplicate runs of the SAME operation, but + # different operations can run concurrently — wishlist downloads + # use bandwidth, watchlist scans use API calls, library scans use + # media-server CPU. Different resources, no contention. + engine.register_action_handler( + 'process_wishlist', + lambda config: auto_process_wishlist(config, deps), + guard_fn=deps.is_wishlist_actually_processing, + ) + engine.register_action_handler( + 'scan_watchlist', + lambda config: auto_scan_watchlist(config, deps), + guard_fn=deps.is_watchlist_actually_scanning, + ) + engine.register_action_handler( + 'scan_library', + lambda config: auto_scan_library(config, deps), + deps.state.is_scan_library_active, + ) + + # Playlist lifecycle handlers. The pipeline composes refresh + + # sync + discover (it imports them directly), so all four ship + # together. The pipeline guard prevents an in-flight pipeline + # from being re-triggered mid-run. + engine.register_action_handler( + 'refresh_mirrored', + lambda config: auto_refresh_mirrored(config, deps), + ) + engine.register_action_handler( + 'sync_playlist', + lambda config: auto_sync_playlist(config, deps), + ) + engine.register_action_handler( + 'discover_playlist', + lambda config: auto_discover_playlist(config, deps), + ) + engine.register_action_handler( + 'playlist_pipeline', + lambda config: auto_playlist_pipeline(config, deps), + deps.state.is_pipeline_running, + ) + # Personalized pipeline shares the pipeline_running flag with the + # mirrored pipeline so the two can't overlap (single sync queue, + # single wishlist worker). + engine.register_action_handler( + 'personalized_pipeline', + lambda config: auto_personalized_pipeline(config, deps), + deps.state.is_pipeline_running, + ) + + # Database update + deep scan share the db_update_state guard — + # only one operation can mutate that state at a time. + engine.register_action_handler( + 'start_database_update', + lambda config: auto_start_database_update(config, deps), + lambda: deps.get_db_update_state().get('status') == 'running', + ) + engine.register_action_handler( + 'deep_scan_library', + lambda config: auto_deep_scan_library(config, deps), + lambda: deps.get_db_update_state().get('status') == 'running', + ) + engine.register_action_handler( + 'run_duplicate_cleaner', + lambda config: auto_run_duplicate_cleaner(config, deps), + lambda: deps.get_duplicate_cleaner_state().get('status') == 'running', + ) + engine.register_action_handler( + 'clear_quarantine', + lambda config: auto_clear_quarantine(config, deps), + ) + engine.register_action_handler( + 'cleanup_wishlist', + lambda config: auto_cleanup_wishlist(config, deps), + ) + engine.register_action_handler( + 'update_discovery_pool', + lambda config: auto_update_discovery_pool(config, deps), + ) + engine.register_action_handler( + 'start_quality_scan', + lambda config: auto_start_quality_scan(config, deps), + lambda: deps.get_quality_scanner_state().get('status') == 'running', + ) + engine.register_action_handler( + 'backup_database', + lambda config: auto_backup_database(config, deps), + ) + engine.register_action_handler( + 'refresh_beatport_cache', + lambda config: auto_refresh_beatport_cache(config, deps), + ) + engine.register_action_handler( + 'clean_search_history', + lambda config: auto_clean_search_history(config, deps), + ) + engine.register_action_handler( + 'clean_completed_downloads', + lambda config: auto_clean_completed_downloads(config, deps), + ) + engine.register_action_handler( + 'full_cleanup', + lambda config: auto_full_cleanup(config, deps), + ) + engine.register_action_handler( + 'run_script', + lambda config: auto_run_script(config, deps), + ) + engine.register_action_handler( + 'search_and_download', + lambda config: auto_search_and_download(config, deps), + ) + + # Progress + history callbacks: the engine invokes these around + # each handler run. Lift the closures from + # `web_server._register_automation_handlers` into thin lambdas + # that delegate into the extracted top-level functions. + engine.register_progress_callbacks( + lambda aid, name, action_type: progress_init(aid, name, action_type, deps), + lambda aid, result: progress_finish(aid, result, deps), + deps.update_progress, + lambda aid, result: record_history(aid, result, deps), + ) + + # `library_scan_completed` event: when the media-server scan + # manager finishes a scan, emit the event so any automation can + # trigger off it. No-op when no scan manager is configured. + register_library_scan_completed_emitter(deps) diff --git a/core/automation/handlers/run_script.py b/core/automation/handlers/run_script.py new file mode 100644 index 00000000..2d10a6fb --- /dev/null +++ b/core/automation/handlers/run_script.py @@ -0,0 +1,103 @@ +"""Automation handler: ``run_script`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_run_script`` closure). Runs a user-provided shell or Python +script from the configured scripts directory with bounded timeout + +captured stdout/stderr. Path-traversal guard ensures users can't +escape the scripts directory. + +Environment variables exposed to the script: +- ``SOULSYNC_EVENT``: triggering event type (when fired by an event) +- ``SOULSYNC_AUTOMATION``: automation name +- ``SOULSYNC_SCRIPTS_DIR``: absolute path to the scripts dir +""" + +from __future__ import annotations + +import os +import subprocess as _sp +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +_MAX_TIMEOUT_SECONDS = 300 # Hard cap on user-supplied timeout config. + + +def auto_run_script(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + script_name = config.get('script_name', '') + timeout = min(int(config.get('timeout', 60)), _MAX_TIMEOUT_SECONDS) + automation_id = config.get('_automation_id') + + if not script_name: + return {'status': 'error', 'error': 'No script selected'} + + scripts_dir = deps.docker_resolve_path(deps.config_manager.get('scripts.path', './scripts')) + if not scripts_dir or not os.path.isdir(scripts_dir): + os.makedirs(scripts_dir, exist_ok=True) + return { + 'status': 'error', + 'error': 'Scripts directory is empty. Add scripts to the scripts/ folder.', + } + + script_path = os.path.join(scripts_dir, script_name) + script_path = os.path.realpath(script_path) + + # Security: block path traversal — script must resolve under + # the scripts dir, no symlinks/.. tricks allowed out. + if not script_path.startswith(os.path.realpath(scripts_dir)): + return {'status': 'error', 'error': 'Script path traversal blocked'} + + if not os.path.isfile(script_path): + return {'status': 'error', 'error': f'Script not found: {script_name}'} + + deps.update_progress(automation_id, phase=f'Running {script_name}...', progress=10) + + # Build environment with SoulSync context. + env = os.environ.copy() + event_data = config.get('_event_data') or {} + env['SOULSYNC_EVENT'] = str(event_data.get('type', '')) + env['SOULSYNC_AUTOMATION'] = config.get('_automation_name', '') + env['SOULSYNC_SCRIPTS_DIR'] = scripts_dir + + try: + # Determine how to run the script. + if script_path.endswith('.py'): + cmd = ['python', script_path] + elif script_path.endswith('.sh'): + cmd = ['bash', script_path] + else: + cmd = [script_path] + + result = _sp.run( + cmd, + capture_output=True, text=True, timeout=timeout, + cwd=scripts_dir, env=env, + ) + + deps.update_progress(automation_id, phase='Script completed', progress=100) + + stdout = result.stdout[:2000] if result.stdout else '' + stderr = result.stderr[:1000] if result.stderr else '' + + if result.returncode == 0: + deps.logger.info(f"Script '{script_name}' completed (exit 0)") + else: + deps.logger.warning(f"Script '{script_name}' exited with code {result.returncode}") + + return { + 'status': 'completed' if result.returncode == 0 else 'error', + 'exit_code': str(result.returncode), + 'stdout': stdout, + 'stderr': stderr, + 'script': script_name, + } + except _sp.TimeoutExpired: + deps.update_progress(automation_id, phase='Script timed out', progress=100) + return { + 'status': 'error', + 'error': f'Script timed out after {timeout}s', + 'script': script_name, + } + except Exception as e: # noqa: BLE001 — automation handlers must never raise + return {'status': 'error', 'error': str(e), 'script': script_name} diff --git a/core/automation/handlers/scan_library.py b/core/automation/handlers/scan_library.py new file mode 100644 index 00000000..824d293f --- /dev/null +++ b/core/automation/handlers/scan_library.py @@ -0,0 +1,158 @@ +"""Automation handler: ``scan_library`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_scan_library`` closure). The handler triggers a media-server +scan via ``web_scan_manager``, then polls the manager's status until +the scan completes (or a 30-minute timeout fires). Progress phases +are emitted via :func:`AutomationDeps.update_progress` so the +trigger card stays current throughout the run. + +The handler manages its own progress reporting (it sets +``_manages_own_progress: True`` in the result) so the engine doesn't +overwrite the live phase string with a generic 'completed' label. +""" + +from __future__ import annotations + +import time +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +# Outer poll cap — covers extreme worst case (long Plex scans on +# huge libraries). Past this point we surface a clear timeout error +# so users notice rather than letting the trigger hang forever. +_SCAN_TIMEOUT_SECONDS = 1800 + +# Per-phase poll intervals. +_POLL_SCHEDULED_SECONDS = 2 +_POLL_SCANNING_SECONDS = 5 +_POLL_UNKNOWN_SECONDS = 2 + +# Progress percentage waypoints. +_PROGRESS_SCHEDULED_MAX = 14 +_PROGRESS_SCAN_START = 15 +_PROGRESS_SCAN_MAX = 95 + + +def auto_scan_library(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run a media-server library scan and stream progress to the + trigger card. + + Returns one of: + - ``{'status': 'completed', '_manages_own_progress': True, ...}`` + - ``{'status': 'skipped', 'reason': 'Scan already being tracked'}`` + - ``{'status': 'error', 'reason': '...', '_manages_own_progress': True}`` + """ + automation_id = config.get('_automation_id') + + if not deps.web_scan_manager: + return {'status': 'error', 'reason': 'Scan manager not available'} + + # If another automation is already tracking the scan, just forward + # the request — the original tracker keeps emitting progress. + if deps.state.is_scan_library_active(): + deps.web_scan_manager.request_scan('Automation trigger (additional batch)') + return {'status': 'skipped', 'reason': 'Scan already being tracked'} + + deps.state.set_scan_library_id(automation_id) + + try: + result = deps.web_scan_manager.request_scan('Automation trigger') + scan_status_val = result.get('status', 'unknown') + + if scan_status_val == 'queued': + deps.update_progress( + automation_id, + log_line='Scan already in progress — waiting for completion', + log_type='info', + ) + else: + delay = result.get('delay_seconds', 60) + deps.update_progress( + automation_id, + log_line=f'Scan scheduled (debounce: {delay}s)', + log_type='info', + ) + + # Unified polling loop — handles debounce → scanning → idle. + poll_start = time.time() + scan_started = (scan_status_val == 'queued') + while time.time() - poll_start < _SCAN_TIMEOUT_SECONDS: + status = deps.web_scan_manager.get_scan_status() + st = status.get('status') + + if st == 'idle': + break # Scan completed (or finished before we polled) + + if st == 'scheduled': + elapsed = int(time.time() - poll_start) + deps.update_progress( + automation_id, + phase=f'Waiting for scan to start... ({elapsed}s)', + progress=min(int(elapsed / 60 * 10), _PROGRESS_SCHEDULED_MAX), + ) + time.sleep(_POLL_SCHEDULED_SECONDS) + continue + + if st == 'scanning': + if not scan_started: + scan_started = True + deps.update_progress( + automation_id, + progress=_PROGRESS_SCAN_START, + log_line='Scan triggered on media server', + log_type='success', + ) + elapsed = status.get('elapsed_seconds', 0) + max_time = status.get('max_time_seconds', 300) + pct = min(_PROGRESS_SCAN_START + int(elapsed / max_time * 80), _PROGRESS_SCAN_MAX) + mins, secs = divmod(elapsed, 60) + deps.update_progress( + automation_id, + phase=f'Library scan in progress... ({mins}m {secs}s)', + progress=pct, + ) + time.sleep(_POLL_SCANNING_SECONDS) + continue + + time.sleep(_POLL_UNKNOWN_SECONDS) + else: + # 30-min timeout reached + deps.update_progress( + automation_id, + status='error', + phase='Timed out', + log_line='Library scan timed out after 30 minutes', + log_type='error', + ) + return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} + + elapsed = round(time.time() - poll_start, 1) + deps.update_progress( + automation_id, + status='finished', + progress=100, + phase='Complete', + log_line='Library scan completed', + log_type='success', + ) + return { + 'status': 'completed', + '_manages_own_progress': True, + 'scan_duration_seconds': elapsed, + } + + except Exception as e: # noqa: BLE001 — automation handlers must never raise into the engine + deps.update_progress( + automation_id, + status='error', + phase='Error', + log_line=str(e), + log_type='error', + ) + return {'status': 'error', 'error': str(e), '_manages_own_progress': True} + + finally: + deps.state.set_scan_library_id(None) diff --git a/core/automation/handlers/scan_watchlist.py b/core/automation/handlers/scan_watchlist.py new file mode 100644 index 00000000..fe522f62 --- /dev/null +++ b/core/automation/handlers/scan_watchlist.py @@ -0,0 +1,46 @@ +"""Automation handler: ``scan_watchlist`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_scan_watchlist`` closure). The watchlist scanner returns +summary stats for the trigger card only when a fresh scan actually +ran — detected by snapshotting ``id(state_dict)`` before/after, since +the live processor reassigns the dict on each new scan. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_scan_watchlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Run a watchlist scan when the automation triggers. + + Pre-scan we capture ``id(watchlist_scan_state)`` so we can tell + afterwards whether the worker ran (and reassigned the state dict) + or short-circuited (kept the same dict). Only fresh scans report + summary stats — repeat triggers without an intervening run return + a bare ``completed``. + """ + try: + pre_state = deps.get_watchlist_scan_state() + pre_state_id = id(pre_state) + deps.process_watchlist_scan_automatically( + automation_id=config.get('_automation_id'), + profile_id=config.get('_profile_id'), + ) + post_state = deps.get_watchlist_scan_state() + # Fresh scan = state dict was reassigned mid-run. + if id(post_state) != pre_state_id: + summary = post_state.get('summary', {}) if isinstance(post_state, dict) else {} + return { + 'status': 'completed', + 'artists_scanned': summary.get('total_artists', 0), + 'successful_scans': summary.get('successful_scans', 0), + 'new_tracks_found': summary.get('new_tracks_found', 0), + 'tracks_added_to_wishlist': summary.get('tracks_added_to_wishlist', 0), + } + return {'status': 'completed'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise into the engine + return {'status': 'error', 'error': str(e)} diff --git a/core/automation/handlers/search_and_download.py b/core/automation/handlers/search_and_download.py new file mode 100644 index 00000000..fa49c534 --- /dev/null +++ b/core/automation/handlers/search_and_download.py @@ -0,0 +1,57 @@ +"""Automation handler: ``search_and_download`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_search_and_download`` closure). Searches for a track by +name/artist string and dispatches the best match through the +download orchestrator. Query can come from the trigger config +(direct value) or from event data (e.g. webhook payload). +""" + +from __future__ import annotations + +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_search_and_download(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + automation_id = config.get('_automation_id') + query = config.get('query', '').strip() + # Event-triggered: pull query from event data (e.g. webhook_received). + if not query: + event_data = config.get('_event_data', {}) + query = (event_data.get('query', '') or '').strip() + if not query: + if automation_id: + deps.update_progress( + automation_id, log_line='No search query provided', log_type='error', + ) + return {'status': 'error', 'error': 'No search query provided'} + try: + if automation_id: + deps.update_progress( + automation_id, phase='Searching', + log_line=f'Searching: {query}', log_type='info', + ) + result = deps.run_async(deps.download_orchestrator.search_and_download_best(query)) + if result: + if automation_id: + deps.update_progress( + automation_id, + log_line=f'Download started for: {query}', + log_type='success', + ) + return {'status': 'completed', 'query': query, 'download_id': result} + if automation_id: + deps.update_progress( + automation_id, + log_line=f'No match found for: {query}', + log_type='warning', + ) + return {'status': 'not_found', 'query': query, 'error': 'No match found'} + except Exception as e: # noqa: BLE001 — automation handlers must never raise + if automation_id: + deps.update_progress( + automation_id, log_line=f'Error: {e}', log_type='error', + ) + return {'status': 'error', 'query': query, 'error': str(e)} diff --git a/core/automation/handlers/sync_playlist.py b/core/automation/handlers/sync_playlist.py new file mode 100644 index 00000000..5734b48f --- /dev/null +++ b/core/automation/handlers/sync_playlist.py @@ -0,0 +1,195 @@ +"""Automation handler: ``sync_playlist`` action. + +Lifted from ``web_server._register_automation_handlers`` (the +``_auto_sync_playlist`` closure). Syncs a mirrored playlist to the +configured media server, using discovered metadata when available +and skipping undiscovered tracks. When triggered on a schedule with +no track changes since the last sync, short-circuits with +``status: skipped`` (saves Plex / Jellyfin / Navidrome from +needless rewrites).""" + +from __future__ import annotations + +import hashlib +import json +import threading +from typing import Any, Dict + +from core.automation.deps import AutomationDeps + + +def auto_sync_playlist(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: + """Sync a mirrored playlist to the active media server. + + Behavior: + - Tracks with discovered metadata (extra_data.discovered + matched_data) + are routed via the official metadata. + - Tracks with a Spotify hint (real Spotify ID from the embed + scraper) are included so they can still hit Soulseek + the + wishlist. + - Tracks with neither are counted as ``skipped_tracks``. + - Empty result → ``status: skipped`` with the skipped count. + - Same track set as last sync (matched_tracks unchanged) → + ``status: skipped`` (no-op). + - Otherwise spawns a daemon thread running ``run_sync_task`` and + returns ``status: started`` with ``_manages_own_progress: True``. + """ + auto_id = config.get('_automation_id') + playlist_id = config.get('playlist_id') + if not playlist_id: + return {'status': 'error', 'reason': 'No playlist specified'} + + db = deps.get_database() + pl = db.get_mirrored_playlist(int(playlist_id)) + if not pl: + return {'status': 'error', 'reason': 'Playlist not found'} + + tracks = db.get_mirrored_playlist_tracks(int(playlist_id)) + if not tracks: + return {'status': 'error', 'reason': 'No tracks in playlist'} + + # Convert mirrored tracks to format expected by run_sync_task. + # Use discovered metadata when available, fall back to Spotify + # hint or raw playlist fields when not. + tracks_json = [] + skipped_count = 0 + + for t in tracks: + # Parse extra_data for discovery info. + extra = {} + if t.get('extra_data'): + try: + extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] + except (json.JSONDecodeError, TypeError): + pass + + if extra.get('discovered') and extra.get('matched_data'): + # Use official discovered metadata. + md = extra['matched_data'] + album_raw = md.get('album', '') + album_obj = album_raw if isinstance(album_raw, dict) else {'name': album_raw or ''} + _track_entry = { + 'name': md.get('name', ''), + 'artists': md.get('artists', [{'name': t.get('artist_name', '')}]), + 'album': album_obj, + 'duration_ms': md.get('duration_ms', 0), + 'id': md.get('id', ''), + } + if md.get('track_number'): + _track_entry['track_number'] = md['track_number'] + if md.get('disc_number'): + _track_entry['disc_number'] = md['disc_number'] + tracks_json.append(_track_entry) + else: + # NOT discovered — try to include using available metadata so + # the track can still be searched on Soulseek and added to + # wishlist. Without this, failed discovery blocks the entire + # download pipeline. + # + # Priority: spotify_hint (has real Spotify ID from embed + # scraper) > raw playlist fields (only if source_track_id + # is valid). + hint = extra.get('spotify_hint', {}) + # Build album object with cover art from the mirrored playlist track. + track_image = (t.get('image_url') or '').strip() + album_obj = { + 'name': (t.get('album_name') or '').strip(), + 'images': [{'url': track_image, 'height': 300, 'width': 300}] if track_image else [], + } + + if hint.get('id') and hint.get('name'): + # spotify_hint has proper Spotify track ID + metadata from embed scraper. + hint_artists = hint.get('artists', []) + if hint_artists and isinstance(hint_artists[0], str): + hint_artists = [{'name': a} for a in hint_artists] + elif hint_artists and isinstance(hint_artists[0], dict): + pass # Already in correct format + else: + hint_artists = [{'name': t.get('artist_name', '')}] + tracks_json.append({ + 'name': hint['name'], + 'artists': hint_artists, + 'album': album_obj, + 'duration_ms': t.get('duration_ms', 0), + 'id': hint['id'], + }) + elif t.get('source_track_id') and (t.get('track_name') or '').strip(): + # Has a valid source ID and track name — usable for wishlist. + tracks_json.append({ + 'name': t['track_name'].strip(), + 'artists': [{'name': (t.get('artist_name') or '').strip() or 'Unknown Artist'}], + 'album': album_obj, + 'duration_ms': t.get('duration_ms', 0), + 'id': t['source_track_id'], + }) + else: + skipped_count += 1 # No usable ID or name — truly can't process. + + if not tracks_json: + deps.update_progress( + auto_id, + log_line=f'No discovered tracks — {skipped_count} need discovery first', + log_type='skip', + ) + return { + 'status': 'skipped', + 'reason': f'No discovered tracks to sync ({skipped_count} tracks need discovery first)', + 'skipped_tracks': str(skipped_count), + } + + # Preflight: hash the track list and compare against last sync. + # Skip if the exact same set of tracks was already synced and + # everything matched (no-op preserves Plex / Jellyfin / Navidrome + # from needless rewrites). + track_ids_str = ','.join(sorted(t.get('id', '') for t in tracks_json)) + tracks_hash = hashlib.md5(track_ids_str.encode()).hexdigest() + + sync_id_key = f"auto_mirror_{playlist_id}" + try: + sync_statuses = deps.load_sync_status_file() + last_status = sync_statuses.get(sync_id_key, {}) + last_hash = last_status.get('tracks_hash', '') + last_matched = last_status.get('matched_tracks', -1) + + if last_hash == tracks_hash and last_matched >= len(tracks_json): + # Exact same tracks, all matched last time — nothing to do. + deps.update_progress( + auto_id, + log_line=f'All {len(tracks_json)} tracks unchanged since last sync — skipping', + log_type='skip', + ) + return { + 'status': 'skipped', + 'reason': f'All {len(tracks_json)} tracks unchanged since last sync', + } + except Exception as e: + deps.logger.debug("mirror sync last-status read: %s", e) + + deps.update_progress( + auto_id, + progress=50, + phase=f'Syncing "{pl["name"]}"', + log_line=f'{len(tracks_json)} discovered, {skipped_count} skipped', + log_type='info', + ) + + sync_id = f"auto_mirror_{playlist_id}" + deps.update_progress( + auto_id, + progress=90, + log_line=f'Starting sync: {len(tracks_json)} tracks', + log_type='success', + ) + threading.Thread( + target=deps.run_sync_task, + args=(sync_id, pl['name'], tracks_json, auto_id, 1, pl.get('image_url', '')), + daemon=True, + name=f'auto-sync-{playlist_id}', + ).start() + return { + 'status': 'started', + 'playlist_name': pl['name'], + 'discovered_tracks': str(len(tracks_json)), + 'skipped_tracks': str(skipped_count), + '_manages_own_progress': True, + } diff --git a/core/discovery/hero.py b/core/discovery/hero.py index 50800ecb..41143924 100644 --- a/core/discovery/hero.py +++ b/core/discovery/hero.py @@ -17,10 +17,14 @@ logger = logging.getLogger(__name__) def get_current_profile_id() -> int: - """Mirror of web_server.get_current_profile_id — uses Flask g.""" + """Mirror of web_server.get_current_profile_id — uses Flask g. + + Catches RuntimeError too because reading `g` outside a request + context raises that (not AttributeError) — happens when this is + called from background threads (sync, automation, scanners).""" try: return g.profile_id - except AttributeError: + except (AttributeError, RuntimeError): return 1 diff --git a/core/discovery/sync.py b/core/discovery/sync.py index 7d63a51f..a77b5d3e 100644 --- a/core/discovery/sync.py +++ b/core/discovery/sync.py @@ -397,9 +397,12 @@ def run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, p } logger.info(f"Sync finished for {playlist_id} - state updated") - # Set playlist poster image if available (Plex, Jellyfin, Emby) + # Set playlist poster image if available (Plex, Jellyfin, Emby). + # Don't log the URL itself — it may carry an auth token (Plex + # X-Plex-Token / Jellyfin X-Emby-Token / Subsonic auth) that we + # don't want persisted to app.log. _synced = getattr(result, 'synced_tracks', 0) - logger.info(f"[PLAYLIST IMAGE] image_url={playlist_image_url!r}, synced_tracks={_synced}") + logger.info(f"[PLAYLIST IMAGE] has_image={bool(playlist_image_url)}, synced_tracks={_synced}") if playlist_image_url and _synced > 0: try: active_server = deps.config_manager.get_active_media_server() diff --git a/core/download_orchestrator.py b/core/download_orchestrator.py index e8231e09..480ce7d6 100644 --- a/core/download_orchestrator.py +++ b/core/download_orchestrator.py @@ -104,6 +104,15 @@ class DownloadOrchestrator: deezer_dl.reconnect(deezer_arl) deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac') + # Reload Amazon quality preference (T2Tunes needs no reconnect — public proxy) + amazon = self.client('amazon') + if amazon: + quality = config_manager.get('amazon_download.quality', 'flac') + amazon._quality = quality + amazon._allow_fallback = config_manager.get('amazon_download.allow_fallback', True) + if hasattr(amazon, '_client') and amazon._client: + amazon._client.preferred_codec = quality + # Reload download path for all clients that cache it. # Soulseek owns the path config and is reloaded above; every # other source mirrors that path so files all land in one @@ -319,7 +328,7 @@ class DownloadOrchestrator: return None # 2. Filter and validate results - _streaming_sources = ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') + _streaming_sources = ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon') is_streaming = tracks[0].username in _streaming_sources if tracks else False if is_streaming and expected_track: diff --git a/core/download_plugins/registry.py b/core/download_plugins/registry.py index ed62e899..ad01c2e9 100644 --- a/core/download_plugins/registry.py +++ b/core/download_plugins/registry.py @@ -38,6 +38,7 @@ from core.download_plugins.base import DownloadSourcePlugin # than the legacy module-top imports here. Importing everything at # registry-load time pins the bindings the same way the legacy # orchestrator did. +from core.amazon_download_client import AmazonDownloadClient from core.deezer_download_client import DeezerDownloadClient from core.hifi_client import HiFiClient from core.lidarr_download_client import LidarrDownloadClient @@ -176,6 +177,7 @@ def build_default_registry() -> DownloadPluginRegistry: """ registry = DownloadPluginRegistry() + registry.register(PluginSpec(name='amazon', factory=AmazonDownloadClient, display_name='Amazon Music')) registry.register(PluginSpec(name='soulseek', factory=SoulseekClient, display_name='Soulseek')) registry.register(PluginSpec(name='youtube', factory=YouTubeClient, display_name='YouTube')) registry.register(PluginSpec(name='tidal', factory=TidalDownloadClient, display_name='Tidal')) diff --git a/core/downloads/candidates.py b/core/downloads/candidates.py index d74cfa9d..edb1fdc1 100644 --- a/core/downloads/candidates.py +++ b/core/downloads/candidates.py @@ -235,7 +235,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, # 1. Try track_info (from frontend, has album track data) tn = track_info.get('track_number', 0) if isinstance(track_info, dict) else 0 - dn = track_info.get('disc_number', 1) if isinstance(track_info, dict) else 1 + dn = (track_info.get('disc_number') or 1) if isinstance(track_info, dict) else 1 if tn and tn > 0: enhanced_payload['track_number'] = tn enhanced_payload['disc_number'] = dn @@ -255,7 +255,7 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, detailed_track = deps.spotify_client.get_track_details(track.id) if detailed_track and detailed_track.get('track_number'): enhanced_payload['track_number'] = detailed_track['track_number'] - enhanced_payload['disc_number'] = detailed_track.get('disc_number', 1) + enhanced_payload['disc_number'] = detailed_track.get('disc_number') or 1 got_track_number = True logger.info(f"[Context] Added track_number from API: {detailed_track['track_number']}, disc_number: {enhanced_payload['disc_number']}") diff --git a/core/downloads/master.py b/core/downloads/master.py index 40a29ab2..3e32cc65 100644 --- a/core/downloads/master.py +++ b/core/downloads/master.py @@ -483,7 +483,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma # Use ALL tracks (tracks_json), not just missing ones, to correctly detect multi-disc # even when only one disc has missing tracks if batch_is_album and batch_album_context: - total_discs = max((t.get('disc_number', 1) for t in tracks_json), default=1) + total_discs = max((t.get('disc_number') or 1 for t in tracks_json), default=1) batch_album_context['total_discs'] = total_discs if total_discs > 1: logger.info(f"[Multi-Disc] Detected {total_discs} discs for album '{batch_album_context.get('name')}'") @@ -507,7 +507,7 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma # Fallback album key: use album name when ID is missing (e.g. mirrored playlist tracks) if not album_id and isinstance(album_val, dict) and album_val.get('name'): album_id = f"_name_{album_val['name'].lower().strip()}" - disc_num = sp_data.get('disc_number', t.get('disc_number', 1)) + disc_num = sp_data.get('disc_number') or t.get('disc_number') or 1 if album_id: wishlist_album_disc_counts[album_id] = max( wishlist_album_disc_counts.get(album_id, 1), disc_num diff --git a/core/downloads/status.py b/core/downloads/status.py index c1599c5a..e9b8d925 100644 --- a/core/downloads/status.py +++ b/core/downloads/status.py @@ -87,7 +87,7 @@ class StatusDeps: # Streaming sources the engine fallback applies to. Soulseek goes through # slskd's live_transfers path and must NOT hit the engine fallback. _STREAMING_SOURCE_NAMES = frozenset(( - 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', + 'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon', )) # Keep these in sync with the engine plugins' state strings. diff --git a/core/downloads/task_worker.py b/core/downloads/task_worker.py index ebe5f716..c5d79d82 100644 --- a/core/downloads/task_worker.py +++ b/core/downloads/task_worker.py @@ -294,7 +294,7 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke source_clients = { name: orch.client(name) for name in ('soulseek', 'youtube', 'tidal', 'qobuz', - 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') + 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon') } # The orchestrator tried sources in order but stopped at the first with results. diff --git a/core/downloads/validation.py b/core/downloads/validation.py index 17171535..79131472 100644 --- a/core/downloads/validation.py +++ b/core/downloads/validation.py @@ -79,7 +79,7 @@ def get_valid_candidates(results, spotify_track, query): # Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer, SoundCloud) return structured API results # with proper artist/title metadata — score using the same matching engine as Soulseek - _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud") + _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl", "soundcloud", "amazon") if results[0].username in _streaming_sources: source_label = results[0].username.replace('_dl', '').title() expected_artists = spotify_track.artists if spotify_track else [] diff --git a/core/imports/quarantine.py b/core/imports/quarantine.py index 7c4488c6..5b25e299 100644 --- a/core/imports/quarantine.py +++ b/core/imports/quarantine.py @@ -130,6 +130,16 @@ def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]: except OSError: size_bytes = 0 + # Issue #608 follow-up (AfonsoG6): surface the source username + # + filename that was originally downloaded, so the user can see + # at a glance which uploader the bad file came from. Lives + # under `context.original_search_result` when full context is + # persisted; absent on legacy thin sidecars. + ctx = sidecar.get("context") if isinstance(sidecar.get("context"), dict) else {} + osr = ctx.get("original_search_result") if isinstance(ctx.get("original_search_result"), dict) else {} + source_username = osr.get("username", "") if isinstance(osr, dict) else "" + source_filename = osr.get("filename", "") if isinstance(osr, dict) else "" + entries.append( { "id": entry_id, @@ -142,6 +152,8 @@ def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]: "size_bytes": size_bytes, "has_full_context": isinstance(sidecar.get("context"), dict), "trigger": sidecar.get("trigger", "unknown"), + "source_username": source_username, + "source_filename": source_filename, } ) diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index 0868d6a2..9b3d9f3a 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -235,7 +235,7 @@ def record_library_history_download(context: Dict[str, Any]) -> None: source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "") source_track_title = search_result.get("title", "") or search_result.get("name", "") source_artist = search_result.get("artist", "") - if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr", "soundcloud"): + if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr", "soundcloud", "amazon"): stream_id = source_filename.split("||")[0] if stream_id and not source_track_id: source_track_id = stream_id diff --git a/core/library/reorganize_tag_source.py b/core/library/reorganize_tag_source.py new file mode 100644 index 00000000..de83e9f2 --- /dev/null +++ b/core/library/reorganize_tag_source.py @@ -0,0 +1,327 @@ +"""Build reorganize-planning metadata from a file's embedded tags +instead of from a live metadata-source API call. + +Issue #592 (tacobell444): when a library has been carefully enriched ++ tagged, doing a fresh API lookup at reorganize time can introduce +inconsistencies (provider naming drift, version-mismatches, missing +album-level metadata for niche releases). The user's own embedded +tags are usually the most stable source of truth for an enriched +library — and using them costs zero API calls. + +This module is the pure tag-to-context adapter. It turns the dict +that ``core.library.file_tags.read_embedded_tags`` returns into the +``api_album`` / ``api_track`` shapes that +``library_reorganize._build_post_process_context`` already consumes. +That keeps the downstream pipeline path-builder, post-process +helpers, AcoustID, etc.) completely unchanged: tag-mode just produces +the same input shape via a different upstream route. + +Pure helpers — no IO inside the extractors so every shape is +test-pinnable. The wrapper :func:`read_album_track_from_file` does +the file IO via ``read_embedded_tags`` and then routes through the +extractors. + +Returns ``None`` (extractors) / ``(None, None, reason)`` (wrapper) +when the embedded tags are missing fields essential for reorganize +(track title, album name, or track artist). The plan layer surfaces +that as an unmatched item with a clear reason — same UX as when the +metadata-API call returns no candidate. No silent degradation.""" + +from __future__ import annotations + +import os +import re +from typing import Any, Dict, List, Optional, Tuple + + +# Tokens we accept as valid `releasetype` / `albumtype` values. +# Mirrors the canonical set the rest of the metadata pipeline uses +# (`core/metadata/album_tracks.py:_normalize_album_type`). +_VALID_ALBUM_TYPES = frozenset({'album', 'single', 'ep', 'compilation'}) + + +# Match a 4-digit year anywhere in a date-like string ("2020", +# "2020-01-15", "2020/01/15", "Jan 5, 2020", etc.). +_YEAR_RE = re.compile(r'(\d{4})') + + +# Separators we split a single artist field on to recover a list. +# Mirrors the same separator set ``core/metadata/artist_resolution.py`` +# uses when normalizing soulseek matched-download artist strings. +_ARTIST_SPLIT_RE = re.compile( + r'\s*(?:,|;|/|&| feat\. | feat | ft\. | ft | featuring | x | with )\s*', + re.IGNORECASE, +) + + +def _stringify(value: Any) -> str: + """Coerce an embedded-tag value into a clean string.""" + if value is None: + return '' + return str(value).strip() + + +def _parse_int_first(value: Any) -> Optional[int]: + """Parse a track/disc number that may arrive as ``"5"``, ``"5/12"``, + ``5``, ``5.0`` or even ``"05"``. Returns the leading integer, or + ``None`` when no integer is recoverable. + + Defensive against the trailing-``/N`` shape ID3 stores: ``TRCK = + "5/12"`` means "track 5 of 12", and we want ``5``.""" + if value is None: + return None + if isinstance(value, (int,)): + return value + if isinstance(value, float): + return int(value) + s = _stringify(value) + if not s: + return None + head = s.split('/', 1)[0].strip() + try: + return int(head) + except (TypeError, ValueError): + try: + return int(float(head)) + except (TypeError, ValueError): + return None + + +def _parse_int_total(value: Any) -> Optional[int]: + """Parse the trailing ``N`` of an ID3-style ``"5/12"`` value, or + return the parsed value when it's a plain integer string.""" + if value is None: + return None + if isinstance(value, int): + return value + s = _stringify(value) + if not s: + return None + if '/' in s: + tail = s.split('/', 1)[1].strip() + try: + return int(tail) + except (TypeError, ValueError): + return None + try: + return int(s) + except (TypeError, ValueError): + return None + + +def _normalize_year(value: Any) -> str: + """Extract a 4-digit year from a date-like field. Returns '' when + no year is extractable. Reorganize templates only use the year + portion of release dates, so we don't need to preserve the full + date string.""" + s = _stringify(value) + if not s: + return '' + m = _YEAR_RE.search(s) + return m.group(1) if m else '' + + +def _normalize_album_type(value: Any) -> str: + """Lowercase + validate the ``releasetype`` tag against the canonical + token set. Returns '' for unknown values so the downstream path + builder falls back to its default.""" + s = _stringify(value).lower() + if s in _VALID_ALBUM_TYPES: + return s + return '' + + +def _split_artists(value: Any) -> List[str]: + """Split an artist-string field into a list. Handles common + separators (``,``, ``;``, ``/``, ``&``, ``feat``, ``ft``, ``x``, + ``with``). Strips whitespace, drops empties, dedupes (case- + insensitive) while preserving order.""" + s = _stringify(value) + if not s: + return [] + parts = _ARTIST_SPLIT_RE.split(s) + seen: set = set() + out: List[str] = [] + for p in parts: + cleaned = p.strip() + if not cleaned: + continue + key = cleaned.lower() + if key in seen: + continue + seen.add(key) + out.append(cleaned) + return out + + +def _resolve_track_artists(tags: Dict[str, Any]) -> List[str]: + """Resolve the per-track artist list from embedded tags. Prefers a + multi-value ``artists`` tag (TXXX:Artists / Vorbis ``artists``) + over splitting the single-string ``artist`` tag, which is exactly + the precedence the post-download enrichment uses.""" + artists_value = tags.get('artists') + if artists_value: + # Multi-value tag readers may already have joined with ', '. + # Re-split to recover the list. + parts = _split_artists(artists_value) + if parts: + return parts + return _split_artists(tags.get('artist') or '') + + +def extract_track_meta_from_tags(tags: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Build an ``api_track``-shaped dict from embedded tags. + + Returns ``None`` if essential fields are missing (title or + artist). Caller surfaces that as an unmatched plan item. + + Output shape matches what ``library_reorganize._build_post_process_context`` + consumes (``name`` / ``track_number`` / ``disc_number`` / + ``artists`` / ``duration_ms`` / ``id``).""" + if not isinstance(tags, dict) or not tags: + return None + + title = _stringify(tags.get('title')) + if not title: + return None + + artists = _resolve_track_artists(tags) + if not artists: + return None + + track_number = _parse_int_first(tags.get('tracknumber')) or 1 + disc_number = _parse_int_first(tags.get('discnumber')) or 1 + + return { + 'name': title, + 'title': title, # belt-and-braces — both keys are read downstream + 'track_number': track_number, + 'disc_number': disc_number, + 'artists': [{'name': a} for a in artists], + 'duration_ms': 0, # not derivable from tags alone; set later from `duration` + 'id': '', # tag-mode has no source ID; reorganize doesn't need one + 'uri': '', + } + + +def extract_album_meta_from_tags(tags: Dict[str, Any]) -> Dict[str, Any]: + """Build an ``api_album``-shaped dict from embedded tags. + + Falls back to empty / zero values when fields are missing — the + path builder accepts those and uses its own defaults. The album + name is the only field we can't fall back on; if missing the + caller should treat the track as unmatched (handled by + :func:`read_album_track_from_file`).""" + if not isinstance(tags, dict): + tags = {} + + album_name = _stringify(tags.get('album')) + album_artist = _stringify(tags.get('albumartist') or tags.get('album_artist')) + release_date = _normalize_year(tags.get('date') or tags.get('year') or tags.get('originaldate')) + total_tracks = ( + _parse_int_total(tags.get('totaltracks')) + or _parse_int_total(tags.get('tracktotal')) + or _parse_int_total(tags.get('tracknumber')) # may be "5/12" + or 0 + ) + album_type = _normalize_album_type(tags.get('releasetype')) + + # `total_discs` only comes from explicit total signals: a + # `totaldiscs` tag, or the trailing `/N` of an ID3-style + # `discnumber = "1/2"`. A bare `discnumber = "1"` carries no total + # and must NOT be treated as one (else single-disc albums would + # claim total=1 and the path builder would still skip the + # subfolder, but partial-album cases would underreport). + total_discs = _parse_int_total(tags.get('totaldiscs')) or 0 + discnumber_raw = _stringify(tags.get('discnumber')) + if '/' in discnumber_raw: + explicit_total = _parse_int_total(discnumber_raw) + if explicit_total: + total_discs = max(total_discs, explicit_total) + + return { + 'id': '', + 'album_id': '', + 'name': album_name, + 'title': album_name, + 'release_date': release_date, + 'total_tracks': total_tracks, + 'total_discs': total_discs, + 'image_url': '', + 'images': [], + 'album_artist': album_artist, + 'album_type': album_type, + } + + +def read_album_track_from_file( + file_path: str, + *, + read_embedded_tags_fn=None, +) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]], Optional[str]]: + """Read embedded tags from ``file_path`` and produce + ``(album_meta, track_meta, error_reason)``. + + Returns ``(None, None, reason)`` when the file can't be opened, + has no recognisable tags, or is missing essential fields (title + or artist). The reason string is human-readable and suitable for + surfacing directly in the reorganize preview/error UI. + + Args: + file_path: Resolved on-disk path to the audio file. + read_embedded_tags_fn: Optional override for the tag reader, + used by tests to avoid real mutagen IO. Defaults to + ``core.library.file_tags.read_embedded_tags``.""" + if not file_path or not isinstance(file_path, str): + return None, None, 'No file path on track row.' + + if read_embedded_tags_fn is None: + from core.library.file_tags import read_embedded_tags as _real_reader + read_embedded_tags_fn = _real_reader + + result = read_embedded_tags_fn(file_path) + if not isinstance(result, dict) or not result.get('available'): + reason = (result or {}).get('reason') if isinstance(result, dict) else None + return None, None, reason or 'Could not read embedded tags from file.' + + tags = result.get('tags') or {} + track_meta = extract_track_meta_from_tags(tags) + if track_meta is None: + return None, None, 'Embedded tags missing required title or artist.' + + album_meta = extract_album_meta_from_tags(tags) + if not album_meta.get('name'): + return None, None, 'Embedded tags missing album name.' + + # Promote duration from the file-info block onto the track meta + # so the path builder has a non-zero value if a downstream + # consumer wants it. + duration_seconds = result.get('duration') or 0 + try: + track_meta['duration_ms'] = int(float(duration_seconds) * 1000) + except (TypeError, ValueError): + track_meta['duration_ms'] = 0 + + return album_meta, track_meta, None + + +def normalize_resolved_path(file_path: Optional[str]) -> Optional[str]: + """Defensive wrapper: returns the input only when it points at a + real file. Saves the caller from another ``os.path.exists`` check + in already-noisy code paths.""" + if not file_path: + return None + try: + if not os.path.exists(file_path): + return None + except OSError: + return None + return file_path + + +__all__ = [ + 'extract_track_meta_from_tags', + 'extract_album_meta_from_tags', + 'read_album_track_from_file', + 'normalize_resolved_path', +] diff --git a/core/library/service_search.py b/core/library/service_search.py index fb47d48a..cb8b1874 100644 --- a/core/library/service_search.py +++ b/core/library/service_search.py @@ -19,6 +19,7 @@ tidal_enrichment_worker = None qobuz_enrichment_worker = None discogs_worker = None audiodb_worker = None +amazon_worker = None def init( @@ -31,11 +32,12 @@ def init( qobuz_worker=None, discogs_worker_obj=None, audiodb_worker_obj=None, + amazon_worker_obj=None, ): """Bind enrichment worker handles so the lifted bodies can use them.""" global spotify_enrichment_worker, itunes_enrichment_worker, mb_worker global lastfm_worker, genius_worker, tidal_enrichment_worker - global qobuz_enrichment_worker, discogs_worker, audiodb_worker + global qobuz_enrichment_worker, discogs_worker, audiodb_worker, amazon_worker spotify_enrichment_worker = spotify_worker itunes_enrichment_worker = itunes_worker mb_worker = musicbrainz_worker @@ -45,6 +47,7 @@ def init( qobuz_enrichment_worker = qobuz_worker discogs_worker = discogs_worker_obj audiodb_worker = audiodb_worker_obj + amazon_worker = amazon_worker_obj def _detect_provider(items, client): @@ -293,4 +296,22 @@ def _search_service(service, entity_type, query): 'image': None, 'extra': f"{result.get('strArtist', '')} · {result.get('strAlbum', '')}"}] return [] + elif service == 'amazon': + if not amazon_worker or not amazon_worker.client: + raise ValueError("Amazon worker not initialized") + client = amazon_worker.client + if entity_type == 'artist': + items = client.search_artists(query, limit=8) + return [{'id': str(a.id), 'name': a.name, 'image': a.image_url, + 'extra': ', '.join(a.genres[:3]) if a.genres else ''} for a in items] + elif entity_type == 'album': + items = client.search_albums(query, limit=8) + return [{'id': str(a.id), 'name': a.name, 'image': a.image_url, + 'extra': f"{', '.join(a.artists)} · {a.release_date or ''}"} for a in items] + elif entity_type == 'track': + items = client.search_tracks(query, limit=8) + return [{'id': str(t.id), 'name': t.name, 'image': t.image_url, + 'extra': f"{', '.join(t.artists)} · {t.album or ''}"} for t in items] + return [] + return [] diff --git a/core/library_reorganize.py b/core/library_reorganize.py index 36bbf763..2aea45bb 100644 --- a/core/library_reorganize.py +++ b/core/library_reorganize.py @@ -549,17 +549,139 @@ def load_album_and_tracks(db, album_id): pass +def _plan_from_tags( + album_data: dict, + tracks: List[dict], + resolve_file_path_fn: Optional[Callable[[Optional[str]], Optional[str]]], +) -> dict: + """Tag-mode planner: build per-track ``api_track`` shapes from each + file's own embedded metadata instead of a live source API call. + + Per-track behavior: + - File missing on disk → unmatched with reason. + - Tags missing essentials (title / artist / album) → unmatched + with reason. + - Otherwise matched with the per-file extracted ``api_track`` and + a per-file ``api_album``. The plan stores the FIRST matched + track's album dict on the top-level ``api_album`` field for + backward compatibility with downstream callers; downstream + consumers that need the per-track album shape read it off + ``items[i]['api_album']``. + + Returns the same status / source / api_album / total_discs / items + shape as :func:`plan_album_reorganize`. ``source`` is the literal + string ``'tags'`` so callers can distinguish from API sources.""" + if resolve_file_path_fn is None: + # Without the file-path resolver we can't read anything off + # disk. Return an unmatched plan so callers surface a clear + # error instead of silently returning empty. + reason = 'Tag-mode reorganize requires the file path resolver.' + return { + 'status': 'no_source_id', 'source': None, 'api_album': None, + 'total_discs': 1, + 'items': [{ + 'track': t, 'api_track': None, 'matched': False, + 'reason': reason, + } for t in tracks], + } + + from core.library.reorganize_tag_source import read_album_track_from_file + + items: List[dict] = [] + first_album_meta: Optional[dict] = None + max_disc = 1 + + for track in tracks: + db_path = track.get('file_path') + resolved = resolve_file_path_fn(db_path) if db_path else None + if not resolved: + items.append({ + 'track': track, 'api_track': None, 'api_album': None, + 'matched': False, + 'reason': 'File no longer exists on disk for this track.', + }) + continue + + album_meta, track_meta, err = read_album_track_from_file(resolved) + if err is not None or track_meta is None or album_meta is None: + items.append({ + 'track': track, 'api_track': None, 'api_album': None, + 'matched': False, + 'reason': err or 'Could not extract metadata from embedded tags.', + }) + continue + + if first_album_meta is None: + first_album_meta = album_meta + try: + disc = int(track_meta.get('disc_number') or 1) + except (TypeError, ValueError): + disc = 1 + if disc > max_disc: + max_disc = disc + # Respect an explicit `totaldiscs` tag (or "1/2" disc-number + # form) so a partial-album reorganize (only disc 1 present + # locally) still routes into `Disc 1/` when the file's tags + # know there are 2 discs total. + try: + tagged_total = int(album_meta.get('total_discs') or 0) + except (TypeError, ValueError): + tagged_total = 0 + if tagged_total > max_disc: + max_disc = tagged_total + + items.append({ + 'track': track, + 'api_track': track_meta, + 'api_album': album_meta, + 'matched': True, + 'reason': None, + }) + + if not any(it['matched'] for it in items): + return { + 'status': 'no_source_id', + 'source': 'tags', + 'api_album': None, + 'total_discs': 1, + 'items': items, + } + + return { + 'status': 'planned', + 'source': 'tags', + 'api_album': first_album_meta or {}, + 'total_discs': max_disc, + 'items': items, + } + + def plan_album_reorganize( album_data: dict, tracks: List[dict], primary_source: Optional[str] = None, strict_source: bool = False, + metadata_source: str = 'api', + resolve_file_path_fn: Optional[Callable[[Optional[str]], Optional[str]]] = None, ) -> dict: """Compute the per-track plan for an album reorganize without doing any file IO. Both the actual reorganize orchestrator and the preview endpoint share this so the preview is guaranteed to match what would happen on apply. + ``metadata_source``: + - ``'api'`` (default): query the configured metadata source(s) + for the canonical tracklist (existing behavior). Issues an + API call. + - ``'tags'``: read each file's embedded tags as the source of + truth (issue #592). Zero API calls; trusts the user's + enriched library. + + When ``metadata_source='tags'``, ``resolve_file_path_fn`` MUST be + provided (the planner needs to read the actual files). The + ``primary_source`` and ``strict_source`` params are ignored in + tag mode. + Returns: ``{'status': 'planned' | 'no_source_id' | 'no_tracks', 'source': str | None, @@ -581,6 +703,9 @@ def plan_album_reorganize( 'total_discs': 1, 'items': [], } + if metadata_source == 'tags': + return _plan_from_tags(album_data, tracks, resolve_file_path_fn) + if primary_source is None: try: primary_source = get_primary_source() @@ -720,6 +845,7 @@ def preview_album_reorganize( build_final_path_fn: Callable, primary_source: Optional[str] = None, strict_source: bool = False, + metadata_source: str = 'api', ) -> dict: """Compute the planned destination paths for a reorganize WITHOUT moving any files. The preview UI uses this to show users what the @@ -775,6 +901,8 @@ def preview_album_reorganize( plan = plan_album_reorganize( album_data, tracks, primary_source=primary_source, strict_source=strict_source, + metadata_source=metadata_source, + resolve_file_path_fn=resolve_file_path_fn, ) artist_name = album_data.get('artist_name') or 'Unknown Artist' album_title = album_data.get('title') or 'Unknown Album' @@ -836,8 +964,12 @@ def preview_album_reorganize( item['disc_number'] = int(api_track.get('disc_number') or 1) # Build the same context the orchestrator builds so the path # builder produces the same destination it would on apply. + # Tag-mode plan items carry per-item album metadata; fall back + # to the shared api_album in API mode (where every plan item + # shares the same one). + per_item_album = plan_item.get('api_album') or api_album context = _build_post_process_context( - api_album, api_track, artist_name, album_title, total_discs + per_item_album, api_track, artist_name, album_title, total_discs ) # `_build_final_path_for_track` switches between ALBUM and SINGLE # modes based on `album_info.get('is_album')` — must be passed, @@ -1034,13 +1166,18 @@ def _stage_track(ctx: _RunContext, track_id, title, resolved_src) -> Optional[st return staging_file -def _run_post_process_for_track(ctx: _RunContext, track_id, title, api_track, staging_file) -> Optional[str]: +def _run_post_process_for_track(ctx: _RunContext, track_id, title, api_track, staging_file, *, per_item_api_album=None) -> Optional[str]: """Build the per-track context, hand it to post-processing, and return the final on-disk path it produced. Returns None on any failure (exception, AcoustID rejection, internal skip); the caller - leaves the original file alone.""" + leaves the original file alone. + + ``per_item_api_album`` overrides ``ctx.api_album`` for this track — + used in tag-mode reorganize where each file may carry its own + embedded album metadata.""" + api_album = per_item_api_album if per_item_api_album else ctx.api_album context = _build_post_process_context( - ctx.api_album, api_track, ctx.artist_name, ctx.album_title, ctx.total_discs + api_album, api_track, ctx.artist_name, ctx.album_title, ctx.total_discs ) context_key = f"reorganize_{ctx.album_id}_{track_id}_{uuid.uuid4().hex[:8]}" try: @@ -1138,7 +1275,10 @@ def _process_one_track(ctx: _RunContext, plan_item: dict) -> None: if staging_file is None: return - new_path = _run_post_process_for_track(ctx, track_id, title, plan_item['api_track'], staging_file) + new_path = _run_post_process_for_track( + ctx, track_id, title, plan_item['api_track'], staging_file, + per_item_api_album=plan_item.get('api_album'), + ) if new_path is None: return @@ -1180,6 +1320,7 @@ def reorganize_album( primary_source: Optional[str] = None, strict_source: bool = False, stop_check: Optional[Callable[[], bool]] = None, + metadata_source: str = 'api', ) -> dict: """Run a single album through the post-processing pipeline. @@ -1257,10 +1398,19 @@ def reorganize_album( plan = plan_album_reorganize( album_data, tracks, primary_source=primary_source, strict_source=strict_source, + metadata_source=metadata_source, + resolve_file_path_fn=resolve_file_path_fn, ) if plan['status'] == 'no_source_id': summary['status'] = 'no_source_id' - if _is_unknown_artist(album_data.get('artist_name')) or _looks_like_album_id_title(album_data.get('title')): + summary['source'] = plan.get('source') # 'tags' or None + if plan.get('source') == 'tags': + err_text = ( + f"No tracks of '{album_data.get('title', '?')}' have readable " + "embedded tags (missing title / artist / album, or file unreadable). " + "Switch back to API mode or fix the embedded tags first." + ) + elif _is_unknown_artist(album_data.get('artist_name')) or _looks_like_album_id_title(album_data.get('title')): err_text = ( f"Album '{album_data.get('title', '?')}' has placeholder metadata " "(Unknown Artist or numeric title) — run the 'Fix Unknown Artists' " diff --git a/core/matching/version_mismatch.py b/core/matching/version_mismatch.py new file mode 100644 index 00000000..0b60028c --- /dev/null +++ b/core/matching/version_mismatch.py @@ -0,0 +1,115 @@ +"""Decide when an AcoustID version-annotation mismatch should still +pass verification. + +Issue #607 (AfonsoG6): live recordings were quarantining as +"Version mismatch: expected '... (Live at Venue)' (live) but file is +'Song' (original)" because MusicBrainz often stores the recording +entity with a bare title — the venue / live annotation lives on the +release entity, not the recording. The audio fingerprint correctly +identifies the live recording (live audio has its own distinct +fingerprint), but the title-text comparison flagged it as the wrong +version. + +Strict version mismatching stays for genuinely-different recordings +(instrumental vs vocal, remix vs original, acoustic vs studio) — +those have distinct fingerprints AND MB always annotates them in the +recording title. We only loosen for the **live** direction +specifically, since that's the asymmetry users actually hit: + +- LIVE: MB often stores live recordings with bare titles. The + fingerprint correctly identifies the live recording even when the + recording's text title lacks a "(Live at ...)" annotation. +- INSTRUMENTAL / REMIX / ACOUSTIC / DEMO / etc: MB always carries + the version marker in the recording title. If AcoustID returns an + instrumental for a vocal file query (or vice versa), it's a real + wrong-recording match — the fingerprint matched the instrumental + audio, which IS the wrong file. + +Two-sided version mismatches stay strict — "live" vs "remix" really +are different recordings even if MB titled them similarly.""" + +from __future__ import annotations + + +_BARE_VERSION = 'original' + +# Versions where a one-sided annotation difference is plausibly +# explained by MB metadata gaps rather than a different recording. +# Live recordings are the primary case; venue annotations live on the +# release-track entity, not the recording entity, so the recording can +# be bare-titled even though the audio is genuinely live. +_LIVE_AWARE_VERSIONS = frozenset({'live'}) + + +def is_acceptable_version_mismatch( + expected_version: str, + matched_version: str, + *, + fingerprint_score: float, + title_similarity: float, + artist_similarity: float, + score_threshold: float = 0.85, + title_threshold: float = 0.70, + artist_threshold: float = 0.60, +) -> bool: + """Return True when an expected-vs-matched version annotation + difference is likely a MusicBrainz metadata gap rather than a + genuinely different recording. + + Conditions (all must hold): + + 1. The mismatch is **one-sided AND involves a live-aware version** + — exactly one side is ``'live'`` and the other is bare + ``'original'``. Other version markers (instrumental, remix, + acoustic, etc) carry distinct fingerprints AND are always + annotated in MB's recording title — we don't loosen for them. + 2. Fingerprint score ``>= score_threshold`` (default 0.85). The + AcoustID fingerprint is high-confidence — we trust it. + 3. Bare title similarity ``>= title_threshold`` (default 0.70). + After the version annotation is stripped, the underlying titles + agree. + 4. Artist similarity ``>= artist_threshold`` (default 0.60). Same + artist credit. + + When ``expected_version == matched_version`` returns ``True`` + trivially — no mismatch to decide. + + Examples that ACCEPT (return True): + + - expected=``'live'``, matched=``'original'``, fp=0.95, title=0.95, + artist=1.0 — typical live-recording MB-bare-title case (issue + #607 example 2). + - expected=``'original'``, matched=``'live'`` — same case in the + other direction. + + Examples that REJECT (return False): + + - expected=``'instrumental'``, matched=``'original'`` — fingerprint + matched the instrumental recording; if user asked for vocal, the + file is genuinely wrong. Stays strict regardless of confidence. + - expected=``'remix'``, matched=``'original'`` — same logic. + - expected=``'live'``, matched=``'remix'`` — both versioned, + both different. Real mismatch. + - expected=``'live'``, matched=``'original'``, fp=0.50 — one-sided + live but low confidence. Fall through to FAIL. + - expected=``'live'``, matched=``'original'``, fp=0.95 but title + sim 0.30 — bare titles don't agree. Different song. + """ + if expected_version == matched_version: + return True + + one_sided_live = ( + (expected_version in _LIVE_AWARE_VERSIONS and matched_version == _BARE_VERSION) + or (expected_version == _BARE_VERSION and matched_version in _LIVE_AWARE_VERSIONS) + ) + if not one_sided_live: + return False + + return ( + fingerprint_score >= score_threshold + and title_similarity >= title_threshold + and artist_similarity >= artist_threshold + ) + + +__all__ = ['is_acceptable_version_mismatch'] diff --git a/core/metadata/artwork.py b/core/metadata/artwork.py index d405fa01..98af8431 100644 --- a/core/metadata/artwork.py +++ b/core/metadata/artwork.py @@ -28,6 +28,54 @@ __all__ = [ logger = _create_logger("metadata.artwork") +# Query-string keys whose values must be masked when a media-server +# URL ends up in a log line. Plex uses X-Plex-Token, Jellyfin uses +# X-Emby-Token / api_key, Navidrome's Subsonic auth uses t (token) + +# s (salt) + p (password fallback). Logs end up persisted to disk — +# leaking any of these gives full read access to the user's library. +_REDACT_QUERY_KEYS = ( + 'x-plex-token', 'x-emby-token', 'api_key', 'apikey', + 't', 's', 'p', 'token', 'password', +) +_REDACT_KEYS_ALT = '|'.join(re.escape(k) for k in _REDACT_QUERY_KEYS) +# Plain form: `?key=value` or `&key=value`. Anchored on `?` / `&` (or +# string start) so short keys like `t` only match at parameter +# boundaries — not as a substring of `format=Jpg`. +_REDACT_QUERY_RE = re.compile( + r'(?i)(?P^|[?&])(?P' + _REDACT_KEYS_ALT + r')=(?P[^&\s]+)' +) +# URL-encoded form: `%3Fkey%3Dvalue` or `%26key%3Dvalue`. The image +# proxy wraps the original URL via `?url=`, so the auth +# params end up encoded inside another URL. Without this second pass +# the encoded form survives plain redaction and ships to logs intact. +_REDACT_QUERY_RE_ENCODED = re.compile( + r'(?i)(?P%3F|%26)(?P' + _REDACT_KEYS_ALT + r')%3D(?P[^%&\s]+?)(?=%26|&|\s|$)' +) + + +def _redact_url_secrets(url: str | None) -> str: + """Mask sensitive query parameters in a URL so the result is safe + to log. Handles both the plain form (``?token=abc``) and the URL- + encoded form (``%3Ftoken%3Dabc``) — the latter shows up when an + auth-bearing URL is wrapped inside another URL's query string + (e.g. our `/api/image-proxy?url=` flow). + + Returns ``''`` for None/empty input. Idempotent (safe to call on + already-redacted strings).""" + if not url: + return '' + out = str(url) + out = _REDACT_QUERY_RE.sub( + lambda m: f"{m.group('lead')}{m.group('key')}=***REDACTED***", + out, + ) + out = _REDACT_QUERY_RE_ENCODED.sub( + lambda m: f"{m.group('lead')}{m.group('key')}%3D***REDACTED***", + out, + ) + return out + + def normalize_image_url(thumb_url: str | None) -> str | None: """Convert media-server image URLs into browser-safe URLs.""" if not thumb_url: @@ -62,7 +110,6 @@ def normalize_image_url(thumb_url: str | None) -> str | None: plex_config = cfg.get_plex_config() plex_base_url = plex_config.get('base_url', '') plex_token = plex_config.get('token', '') - logger.info("Plex config - base_url: %s, token: %s...", plex_base_url, plex_token[:10]) if plex_base_url and plex_token: # Extract the path from URL @@ -76,18 +123,13 @@ def normalize_image_url(thumb_url: str | None) -> str | None: # Construct proper Plex URL with token fixed_url = f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}" - logger.info("Fixed URL: %s", fixed_url) + logger.debug("Fixed URL: %s", _redact_url_secrets(fixed_url)) return _browser_safe_image_url(fixed_url) elif active_server == 'jellyfin': jellyfin_config = cfg.get_jellyfin_config() jellyfin_base_url = jellyfin_config.get('base_url', '') jellyfin_token = jellyfin_config.get('api_key', '') - logger.info( - "Jellyfin config - base_url: %s, token: %s...", - jellyfin_base_url, - jellyfin_token[:10] if jellyfin_token else 'None', - ) if jellyfin_base_url: # Extract the path from URL @@ -105,7 +147,7 @@ def normalize_image_url(thumb_url: str | None) -> str | None: fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}" else: fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}" - logger.info("Fixed URL: %s", fixed_url) + logger.debug("Fixed URL: %s", _redact_url_secrets(fixed_url)) return _browser_safe_image_url(fixed_url) elif active_server == 'navidrome': @@ -113,7 +155,6 @@ def normalize_image_url(thumb_url: str | None) -> str | None: navidrome_base_url = navidrome_config.get('base_url', '') navidrome_username = navidrome_config.get('username', '') navidrome_password = navidrome_config.get('password', '') - logger.info("Navidrome config - base_url: %s, username: %s", navidrome_base_url, navidrome_username) if navidrome_base_url and navidrome_username and navidrome_password: # Extract the path from URL @@ -137,7 +178,7 @@ def normalize_image_url(thumb_url: str | None) -> str | None: # Construct proper Navidrome Subsonic URL fixed_url = f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}" - logger.info("Fixed URL: %s", fixed_url) + logger.debug("Fixed URL: %s", _redact_url_secrets(fixed_url)) return _browser_safe_image_url(fixed_url) logger.warning("No configuration found for %s or unsupported server type", active_server) @@ -146,7 +187,7 @@ def normalize_image_url(thumb_url: str | None) -> str | None: return _browser_safe_image_url(thumb_url) except Exception as exc: - logger.error("Error fixing image URL '%s': %s", thumb_url, exc) + logger.error("Error fixing image URL '%s': %s", _redact_url_secrets(thumb_url), exc) return _browser_safe_image_url(thumb_url) diff --git a/core/metadata/registry.py b/core/metadata/registry.py index d2dc86f7..ef7fbc8f 100644 --- a/core/metadata/registry.py +++ b/core/metadata/registry.py @@ -140,6 +140,14 @@ def _get_discogs_factory(client_factory: Optional[MetadataClientFactory]) -> Met return DiscogsClient +def _get_amazon_factory(client_factory: Optional[MetadataClientFactory]) -> MetadataClientFactory: + if client_factory is not None: + return client_factory + from core.amazon_client import AmazonClient + + return AmazonClient + + def get_spotify_client(client_factory: Optional[MetadataClientFactory] = None): """Get shared Spotify client. @@ -260,6 +268,18 @@ def get_discogs_client( return client +def get_amazon_client(client_factory: Optional[MetadataClientFactory] = None): + """Get cached Amazon Music client.""" + cache_key = "amazon" + factory = _get_amazon_factory(client_factory) + with _client_cache_lock: + client = _client_cache.get(cache_key) + if client is None: + client = factory() + _client_cache[cache_key] = client + return client + + def is_hydrabase_enabled() -> bool: """Return True when Hydrabase is connected and app-enabled.""" try: @@ -331,6 +351,7 @@ def get_primary_client( itunes_client_factory: Optional[MetadataClientFactory] = None, deezer_client_factory: Optional[MetadataClientFactory] = None, discogs_client_factory: Optional[MetadataClientFactory] = None, + amazon_client_factory: Optional[MetadataClientFactory] = None, ): """Return client for configured primary source.""" return get_client_for_source( @@ -339,6 +360,7 @@ def get_primary_client( itunes_client_factory=itunes_client_factory, deezer_client_factory=deezer_client_factory, discogs_client_factory=discogs_client_factory, + amazon_client_factory=amazon_client_factory, ) @@ -348,6 +370,7 @@ def get_primary_source_status( itunes_client_factory: Optional[MetadataClientFactory] = None, deezer_client_factory: Optional[MetadataClientFactory] = None, discogs_client_factory: Optional[MetadataClientFactory] = None, + amazon_client_factory: Optional[MetadataClientFactory] = None, ) -> Dict[str, Any]: """Return a generic status snapshot for the active primary metadata source.""" source = _get_config_value("metadata.fallback_source", "deezer") or "deezer" @@ -361,6 +384,7 @@ def get_primary_source_status( itunes_client_factory=itunes_client_factory, deezer_client_factory=deezer_client_factory, discogs_client_factory=discogs_client_factory, + amazon_client_factory=amazon_client_factory, ) if source == "spotify": connected = bool(client and client.is_spotify_authenticated()) @@ -387,6 +411,7 @@ def get_client_for_source( itunes_client_factory: Optional[MetadataClientFactory] = None, deezer_client_factory: Optional[MetadataClientFactory] = None, discogs_client_factory: Optional[MetadataClientFactory] = None, + amazon_client_factory: Optional[MetadataClientFactory] = None, ): """Return exact client for a source, or None if unavailable.""" if source == "spotify": @@ -410,4 +435,7 @@ def get_client_for_source( if source == "itunes": return get_itunes_client(client_factory=itunes_client_factory) + if source == "amazon": + return get_amazon_client(client_factory=amazon_client_factory) + return None diff --git a/core/metadata_service.py b/core/metadata_service.py index 5391a80b..748a9cf9 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -42,6 +42,7 @@ from core.metadata.registry import ( clear_cached_metadata_client, clear_cached_metadata_clients, clear_cached_profile_spotify_client, + get_amazon_client, get_client_for_source, get_deezer_client, get_discogs_client, @@ -75,6 +76,7 @@ except Exception: # pragma: no cover - optional dependency fallback __all__ = [ "METADATA_SOURCE_PRIORITY", + "get_amazon_client", "MetadataCache", "MetadataLookupOptions", "MetadataProvider", diff --git a/core/personalized/__init__.py b/core/personalized/__init__.py new file mode 100644 index 00000000..18190e61 --- /dev/null +++ b/core/personalized/__init__.py @@ -0,0 +1,24 @@ +"""Standardized personalized-playlist subsystem. + +Replaces the patchwork of `PersonalizedPlaylistsService` (computed-on- +demand views, no persistence) + `discovery_curated_playlists` (ID-only +storage) + `curated_seasonal_playlists` (full storage) with a single +unified abstraction: + +- ``manager.PersonalizedPlaylistManager`` — owns the storage layer + + generator dispatch + refresh lifecycle. +- ``specs.PlaylistKindSpec`` — one spec per playlist KIND + (``hidden_gems``, ``time_machine``, ``seasonal_mix``, etc.) with + generator function, default config, variant resolver, and display- + name template. +- ``types.Track`` / ``types.PlaylistConfig`` — shared dataclasses. + +The legacy ``PersonalizedPlaylistsService`` keeps its existing +generator implementations — they're called BY the manager rather than +duplicated. This means: +- The improved diversity logic / popularity thresholds / blacklist + filtering all stays. +- New behavior layered on top: persistence, refresh-on-demand, + per-playlist user-tweakable config, staleness windows, listening- + history cross-reference, seeded randomization. +""" diff --git a/core/personalized/api.py b/core/personalized/api.py new file mode 100644 index 00000000..bfa9de60 --- /dev/null +++ b/core/personalized/api.py @@ -0,0 +1,163 @@ +"""HTTP endpoint handlers for the personalized-playlists subsystem. + +Wired into the Flask app from web_server.py. Each handler is a thin +wrapper that: +1. Pulls profile id + manager from request context. +2. Calls one PersonalizedPlaylistManager method. +3. Returns a JSON-serializable shape. + +Live routes (registered against the main Flask app): +- GET /api/personalized/playlists — list +- GET /api/personalized/kinds — registry +- GET /api/personalized/playlist/ — singleton +- GET /api/personalized/playlist// — variant +- POST /api/personalized/playlist//refresh — singleton +- POST /api/personalized/playlist///refresh — variant +- PUT /api/personalized/playlist//config — singleton +- PUT /api/personalized/playlist///config — variant + +The handlers themselves are pure functions returning Python dicts so +they're testable without spinning up Flask. The wiring step in +web_server.py wraps them in `jsonify` + URL routing. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from core.personalized.manager import PersonalizedPlaylistManager +from core.personalized.specs import PlaylistKindRegistry, get_registry +from core.personalized.types import PlaylistRecord, Track + + +def _record_to_dict(record: PlaylistRecord) -> Dict[str, Any]: + return { + 'id': record.id, + 'profile_id': record.profile_id, + 'kind': record.kind, + 'variant': record.variant, + 'name': record.name, + 'config': record.config.to_json_dict(), + 'track_count': record.track_count, + 'last_generated_at': record.last_generated_at, + 'last_synced_at': record.last_synced_at, + 'last_generation_source': record.last_generation_source, + 'last_generation_error': record.last_generation_error, + } + + +def _track_to_dict(track: Track) -> Dict[str, Any]: + return { + 'spotify_track_id': track.spotify_track_id, + 'itunes_track_id': track.itunes_track_id, + 'deezer_track_id': track.deezer_track_id, + 'track_name': track.track_name, + 'artist_name': track.artist_name, + 'album_name': track.album_name, + 'album_cover_url': track.album_cover_url, + 'duration_ms': track.duration_ms, + 'popularity': track.popularity, + 'track_data_json': track.track_data_json, + 'source': track.source, + } + + +def list_kinds( + registry: Optional[PlaylistKindRegistry] = None, + manager: Optional[PersonalizedPlaylistManager] = None, +) -> Dict[str, Any]: + """Return every registered playlist kind with metadata. + + UI uses this to render the "available playlists" picker. Each + kind reports whether it requires a variant; when a manager is + supplied AND the kind has a variant_resolver, the resolved + variant list is also included so the UI can render variant + checkboxes without a second round-trip per kind.""" + reg = registry or get_registry() + out = [] + for spec in reg.all(): + entry = { + 'kind': spec.kind, + 'name_template': spec.name_template, + 'description': spec.description, + 'requires_variant': spec.requires_variant, + 'tags': list(spec.tags), + 'default_config': spec.default_config.to_json_dict(), + 'variants': [], + } + if manager is not None and spec.variant_resolver is not None: + try: + entry['variants'] = list(spec.variant_resolver(manager.deps) or []) + except Exception: + entry['variants'] = [] + out.append(entry) + return {'success': True, 'kinds': out} + + +def list_playlists(manager: PersonalizedPlaylistManager, profile_id: int) -> Dict[str, Any]: + """List every persisted playlist for a profile.""" + records = manager.list_playlists(profile_id) + return { + 'success': True, + 'playlists': [_record_to_dict(r) for r in records], + } + + +def get_playlist_with_tracks( + manager: PersonalizedPlaylistManager, + kind: str, + variant: str, + profile_id: int, +) -> Dict[str, Any]: + """Get the playlist row + its current track snapshot. Auto-creates + the row from default config if it doesn't exist (so the UI's first- + paint of an unseen kind works without a separate ensure call).""" + record = manager.ensure_playlist(kind, variant, profile_id) + tracks = manager.get_playlist_tracks(record.id) + return { + 'success': True, + 'playlist': _record_to_dict(record), + 'tracks': [_track_to_dict(t) for t in tracks], + } + + +def refresh_playlist( + manager: PersonalizedPlaylistManager, + kind: str, + variant: str, + profile_id: int, + config_overrides: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Run the kind's generator and persist the snapshot. Returns the + fresh row + tracks.""" + record = manager.refresh_playlist(kind, variant, profile_id, config_overrides=config_overrides) + tracks = manager.get_playlist_tracks(record.id) + return { + 'success': True, + 'playlist': _record_to_dict(record), + 'tracks': [_track_to_dict(t) for t in tracks], + } + + +def update_config( + manager: PersonalizedPlaylistManager, + kind: str, + variant: str, + profile_id: int, + overrides: Dict[str, Any], +) -> Dict[str, Any]: + """Patch the playlist's config with the provided fields.""" + record = manager.update_config(kind, variant, profile_id, overrides) + return { + 'success': True, + 'playlist': _record_to_dict(record), + } + + +__all__ = [ + 'list_kinds', + 'list_playlists', + 'get_playlist_with_tracks', + 'refresh_playlist', + 'update_config', +] diff --git a/core/personalized/generators/__init__.py b/core/personalized/generators/__init__.py new file mode 100644 index 00000000..8bd6309a --- /dev/null +++ b/core/personalized/generators/__init__.py @@ -0,0 +1,27 @@ +"""Per-kind generators for the personalized-playlists subsystem. + +Each module in this subpackage: +1. Defines a generator function ``generate(deps, variant, config)`` + that returns a ``List[Track]``. +2. Calls ``get_registry().register(spec)`` at import time so the + manager auto-discovers it. + +The legacy ``core.personalized_playlists.PersonalizedPlaylistsService`` +keeps its existing implementations — the wrappers in this package +just adapt the call surface (`PlaylistConfig` → method kwargs) and +coerce results into ``Track`` instances. + +To register every generator, import this package — `from +core.personalized import generators` — typically done once at +application startup.""" + +# Importing each module triggers its registration side-effect. +from core.personalized.generators import hidden_gems # noqa: F401 +from core.personalized.generators import discovery_shuffle # noqa: F401 +from core.personalized.generators import popular_picks # noqa: F401 +from core.personalized.generators import time_machine # noqa: F401 +from core.personalized.generators import genre_playlist # noqa: F401 +from core.personalized.generators import daily_mix # noqa: F401 +from core.personalized.generators import fresh_tape # noqa: F401 +from core.personalized.generators import archives # noqa: F401 +from core.personalized.generators import seasonal_mix # noqa: F401 diff --git a/core/personalized/generators/_common.py b/core/personalized/generators/_common.py new file mode 100644 index 00000000..1b5d610f --- /dev/null +++ b/core/personalized/generators/_common.py @@ -0,0 +1,37 @@ +"""Shared helpers for personalized-playlist generators. + +Each per-kind generator module is small + mechanical — it pulls the +legacy ``PersonalizedPlaylistsService`` instance off the deps object +and calls the matching method, then coerces results. This module +holds the bits every generator reuses so we don't repeat them +five times.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.types import Track + + +def get_service(deps: Any): + """Pull the ``PersonalizedPlaylistsService`` instance from deps. + + Generators access the service via ``deps.service``. Tests can + pass a fake deps namespace with a ``service`` attribute that + returns a stub. Raises a clear error if the dep isn't wired.""" + service = getattr(deps, 'service', None) or (deps.get('service') if isinstance(deps, dict) else None) + if service is None: + raise RuntimeError( + "Personalized generator deps missing `service` " + "(PersonalizedPlaylistsService instance). Wire it during " + "PersonalizedPlaylistManager construction." + ) + return service + + +def coerce_tracks(rows: List[dict]) -> List[Track]: + """Convert legacy generator output (list of dicts) into Track + instances. Tolerates None / non-list inputs by returning [].""" + if not rows: + return [] + return [Track.from_dict(row) for row in rows if isinstance(row, dict)] diff --git a/core/personalized/generators/archives.py b/core/personalized/generators/archives.py new file mode 100644 index 00000000..9da9a2f3 --- /dev/null +++ b/core/personalized/generators/archives.py @@ -0,0 +1,35 @@ +"""The Archives (Spotify Discover Weekly) generator. + +Same shape as Fresh Tape — read curated track-id list from +``discovery_curated_playlists`` under ``discovery_weekly_`` +(fallback ``discovery_weekly``), hydrate via discovery pool.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators.fresh_tape import _hydrate_curated +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'archives' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + return _hydrate_curated(deps, 'discovery_weekly', config) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='The Archives', + description='Your Spotify Discover Weekly — curated discovery picks.', + default_config=PlaylistConfig(limit=50, max_per_album=5, max_per_artist=10), + generator=generate, + requires_variant=False, + tags=['curated', 'spotify'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/daily_mix.py b/core/personalized/generators/daily_mix.py new file mode 100644 index 00000000..c3c96e57 --- /dev/null +++ b/core/personalized/generators/daily_mix.py @@ -0,0 +1,83 @@ +"""Daily Mix generator — top library genre → discovery picks. + +Variant = rank position as a string ('1' / '2' / '3' / '4'). Each +mix tracks the user's Nth top library genre and returns discovery +picks within it. Top genres recompute at refresh time, so as the +library evolves a mix's underlying genre can shift -- the playlist +metadata records which genre was used at the most recent refresh +so the UI can label the mix accurately. + +Note: previously this kind ambitiously promised 50% library + 50% +discovery. The library half was a stub (`tracks` table has no +source IDs to sync), so the new generator is discovery-only. +A future enhancement can backfill source IDs into library rows +and re-add the hybrid behavior.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'daily_mix' + +# Default rank set — UI surfaces 4 daily mixes by default. +_DEFAULT_RANKS = ('1', '2', '3', '4') + +# Number of top library genres to consider when ranking. +_MAX_TOP_GENRES = 8 + + +def _resolve_genre_for_rank(service, rank: int) -> str: + """Look up the user's Nth-ranked top library genre. Returns the + genre key or '' when no genre at that rank. + + Calls ``service.get_top_genres_from_library(limit=...)`` and + indexes the resulting (genre, count) tuples by 0-based rank. + """ + top = service.get_top_genres_from_library(limit=_MAX_TOP_GENRES) or [] + if rank < 1 or rank > len(top): + return '' + pair = top[rank - 1] + if not pair: + return '' + # `top` is List[Tuple[str, int]] per service signature. + return pair[0] if isinstance(pair, (tuple, list)) else str(pair) + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + try: + rank = int(variant) + except (TypeError, ValueError) as exc: + raise ValueError(f"Daily Mix variant {variant!r} must be a rank int") from exc + genre = _resolve_genre_for_rank(service, rank) + if not genre: + # User's library doesn't have enough genres for this rank. + return [] + rows = service.get_genre_playlist(genre=genre, limit=config.limit) + return coerce_tracks(rows) + + +def variant_resolver(deps: Any) -> List[str]: + """Return the standard rank set.""" + return list(_DEFAULT_RANKS) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Daily Mix {variant}', + description='Personalized mix based on your top library genres. One mix per top genre rank.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=3), + generator=generate, + variant_resolver=variant_resolver, + requires_variant=True, + tags=['discovery', 'personalized'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/discovery_shuffle.py b/core/personalized/generators/discovery_shuffle.py new file mode 100644 index 00000000..daf13509 --- /dev/null +++ b/core/personalized/generators/discovery_shuffle.py @@ -0,0 +1,33 @@ +"""Discovery Shuffle generator — pure-random discovery pool exploration.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'discovery_shuffle' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + rows = service.get_discovery_shuffle(limit=config.limit) + return coerce_tracks(rows) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Discovery Shuffle', + description='Pure random shuffle from the discovery pool — different every refresh.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=2), + generator=generate, + requires_variant=False, + tags=['discovery'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/fresh_tape.py b/core/personalized/generators/fresh_tape.py new file mode 100644 index 00000000..c179bf6d --- /dev/null +++ b/core/personalized/generators/fresh_tape.py @@ -0,0 +1,119 @@ +"""Fresh Tape (Spotify Release Radar) generator. + +Reads the curated track-id list cached in ``discovery_curated_playlists`` +under ``release_radar_`` (with fallback to ``release_radar``) +and hydrates each ID against the discovery pool to produce full Track +records. The Spotify enrichment worker is responsible for keeping the +curated list fresh — this generator is just a read-and-hydrate path.""" + +from __future__ import annotations + +import json +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'fresh_tape' + + +def _hydrate_curated(deps: Any, curated_type_prefix: str, config: PlaylistConfig) -> List[Track]: + """Shared body for Fresh Tape + Archives — pulls the cached IDs + from discovery_curated_playlists and hydrates them via the live + discovery pool. Returns a Track list trimmed to ``config.limit``.""" + # Allow tests to inject a fake db / service; production flow gets + # them from the manager's deps. + db = getattr(deps, 'database', None) or (deps.get('database') if isinstance(deps, dict) else None) + if db is None: + raise RuntimeError("Curated-playlist generator deps missing `database`") + + profile_id = _resolve_profile_id(deps) + active_source = _resolve_active_source(deps) + + # Try source-specific then generic, mirrors web_server endpoint behavior. + curated_ids = ( + db.get_curated_playlist(f'{curated_type_prefix}_{active_source}', profile_id=profile_id) + or db.get_curated_playlist(curated_type_prefix, profile_id=profile_id) + or [] + ) + if not curated_ids: + return [] + + pool_rows = db.get_discovery_pool_tracks( + limit=5000, new_releases_only=False, + source=active_source, profile_id=profile_id, + ) + by_id = {} + for t in pool_rows: + if active_source == 'spotify' and getattr(t, 'spotify_track_id', None): + by_id[t.spotify_track_id] = t + elif active_source == 'deezer' and getattr(t, 'deezer_track_id', None): + by_id[t.deezer_track_id] = t + elif active_source == 'itunes' and getattr(t, 'itunes_track_id', None): + by_id[t.itunes_track_id] = t + + tracks: List[Track] = [] + for tid in curated_ids: + candidate = by_id.get(tid) + if candidate is None: + continue + # The pool track is a row-like object; coerce to dict for + # Track.from_dict's existing tolerance. + td = getattr(candidate, 'track_data_json', None) + if isinstance(td, str): + try: + td = json.loads(td) + except (ValueError, TypeError): + td = None + track_dict = { + 'spotify_track_id': getattr(candidate, 'spotify_track_id', None), + 'itunes_track_id': getattr(candidate, 'itunes_track_id', None), + 'deezer_track_id': getattr(candidate, 'deezer_track_id', None), + 'track_name': getattr(candidate, 'track_name', ''), + 'artist_name': getattr(candidate, 'artist_name', ''), + 'album_name': getattr(candidate, 'album_name', ''), + 'album_cover_url': getattr(candidate, 'album_cover_url', None), + 'duration_ms': getattr(candidate, 'duration_ms', 0), + 'popularity': getattr(candidate, 'popularity', 0), + 'track_data_json': td, + 'source': getattr(candidate, 'source', active_source), + } + tracks.append(Track.from_dict(track_dict)) + if len(tracks) >= config.limit: + break + return tracks + + +def _resolve_profile_id(deps: Any) -> int: + fn = getattr(deps, 'get_current_profile_id', None) or ( + deps.get('get_current_profile_id') if isinstance(deps, dict) else None + ) + return fn() if callable(fn) else 1 + + +def _resolve_active_source(deps: Any) -> str: + fn = getattr(deps, 'get_active_discovery_source', None) or ( + deps.get('get_active_discovery_source') if isinstance(deps, dict) else None + ) + return fn() if callable(fn) else 'spotify' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + return _hydrate_curated(deps, 'release_radar', config) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Fresh Tape', + description='Your Spotify Release Radar — new releases from artists you follow.', + default_config=PlaylistConfig(limit=50, max_per_album=5, max_per_artist=10), + generator=generate, + requires_variant=False, + tags=['curated', 'spotify'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/genre_playlist.py b/core/personalized/generators/genre_playlist.py new file mode 100644 index 00000000..d74b1f94 --- /dev/null +++ b/core/personalized/generators/genre_playlist.py @@ -0,0 +1,78 @@ +"""Genre Playlist generator — discovery picks within one genre. + +Variant = either a parent-genre key from +``PersonalizedPlaylistsService.GENRE_MAPPING`` (e.g. ``'rock'``, +``'electronic_dance'``) or a specific child-genre keyword (e.g. +``'house'``). Stored variant is always normalized to lowercase +underscore-separated form so the UI and storage agree. +""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'genre_playlist' + + +def _normalize_variant_to_genre_key(variant: str, service) -> str: + """Resolve a variant string back into the genre identifier the + legacy service expects. + + Service accepts both parent-genre KEYS from GENRE_MAPPING (e.g. + 'Electronic/Dance', 'Hip Hop/Rap') and free-form keywords. + The URL-safe variant we store is the parent key with `/` replaced + by `_` and lowercased — e.g. 'electronic_dance'. This helper + inverts that mapping.""" + if not variant: + raise ValueError('Genre playlist requires a variant') + + # Build a once-computed lookup of normalized → original parent key. + mapping = getattr(service, 'GENRE_MAPPING', {}) + for parent_key in mapping.keys(): + normalized = parent_key.lower().replace('/', '_').replace(' ', '_') + if normalized == variant.lower(): + return parent_key + + # Fall through: treat the variant as a free-form keyword (the + # legacy service handles partial matching for those). + return variant + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + genre_key = _normalize_variant_to_genre_key(variant, service) + rows = service.get_genre_playlist(genre=genre_key, limit=config.limit) + return coerce_tracks(rows) + + +def variant_resolver(deps: Any) -> List[str]: + """Return the URL-safe variant for every parent genre defined on + the service. Specific (free-form) genre variants aren't enumerated + — they're created on demand when the user requests a custom one.""" + service = get_service(deps) + mapping = getattr(service, 'GENRE_MAPPING', {}) + return [ + parent.lower().replace('/', '_').replace(' ', '_') + for parent in mapping.keys() + ] + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Genre — {variant}', + description='Discovery picks within one genre. Supports parent-genre families + free-form genre keywords.', + default_config=PlaylistConfig(limit=50, max_per_album=3, max_per_artist=5), + generator=generate, + variant_resolver=variant_resolver, + requires_variant=True, + tags=['genre'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/hidden_gems.py b/core/personalized/generators/hidden_gems.py new file mode 100644 index 00000000..5a5f79de --- /dev/null +++ b/core/personalized/generators/hidden_gems.py @@ -0,0 +1,42 @@ +"""Hidden Gems generator — low-popularity tracks from discovery pool. + +Wraps ``PersonalizedPlaylistsService.get_hidden_gems`` so the +existing source-aware popularity threshold + diversity filter +behavior is preserved verbatim. The user-tweakable knobs that +arrive via ``PlaylistConfig`` (limit) flow through; future config +options (popularity_max override, exclude_recent_days) get layered +on the wrapper without changing the legacy implementation.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'hidden_gems' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + rows = service.get_hidden_gems(limit=config.limit) + return coerce_tracks(rows) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Hidden Gems', + description='Low-popularity discovery picks — underground / indie tracks you probably haven\'t heard.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=3), + generator=generate, + requires_variant=False, + tags=['discovery'], +) + + +# Register at import time so the manager auto-discovers this kind. +# Re-import (e.g. test reloads) is tolerated: only register if absent. +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/popular_picks.py b/core/personalized/generators/popular_picks.py new file mode 100644 index 00000000..c95fa16f --- /dev/null +++ b/core/personalized/generators/popular_picks.py @@ -0,0 +1,33 @@ +"""Popular Picks generator — high-popularity discovery pool picks.""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'popular_picks' + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + rows = service.get_popular_picks(limit=config.limit) + return coerce_tracks(rows) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Popular Picks', + description='High-popularity tracks from the discovery pool — what most people are listening to.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=3), + generator=generate, + requires_variant=False, + tags=['discovery'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/seasonal_mix.py b/core/personalized/generators/seasonal_mix.py new file mode 100644 index 00000000..0d09853b --- /dev/null +++ b/core/personalized/generators/seasonal_mix.py @@ -0,0 +1,137 @@ +"""Seasonal Mix generator (variant = season key). + +Variant = season key from ``SEASONAL_CONFIG`` (``'halloween'`` / +``'christmas'`` / ``'valentines'`` / ``'summer'`` / ``'spring'`` / +``'autumn'``). One playlist per season — user picks which seasons +to enable; idle seasons can stay un-refreshed until their active +period. + +Reads curated track IDs from ``curated_seasonal_playlists`` (via +``SeasonalDiscoveryService.get_curated_seasonal_playlist``) and +hydrates them against ``seasonal_tracks`` (which carries full +metadata including ``track_data_json`` for sync-ready downstream +use).""" + +from __future__ import annotations + +import json +from typing import Any, List + +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'seasonal_mix' + + +def _resolve_seasonal_service(deps: Any): + """Pull the SeasonalDiscoveryService instance from deps.""" + svc = getattr(deps, 'seasonal_service', None) or ( + deps.get('seasonal_service') if isinstance(deps, dict) else None + ) + if svc is None: + raise RuntimeError( + "Seasonal mix generator deps missing `seasonal_service` " + "(SeasonalDiscoveryService instance)." + ) + return svc + + +def _resolve_database(deps: Any): + db = getattr(deps, 'database', None) or ( + deps.get('database') if isinstance(deps, dict) else None + ) + if db is None: + raise RuntimeError("Seasonal mix generator deps missing `database`") + return db + + +def _resolve_active_source(deps: Any) -> str: + fn = getattr(deps, 'get_active_discovery_source', None) or ( + deps.get('get_active_discovery_source') if isinstance(deps, dict) else None + ) + return fn() if callable(fn) else 'spotify' + + +def _hydrate_seasonal_tracks(db, season_key: str, source: str, track_ids: List[str]) -> List[Track]: + """Look up the seasonal_tracks rows for the given IDs.""" + if not track_ids: + return [] + placeholders = ','.join('?' * len(track_ids)) + with db._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + f""" + SELECT spotify_track_id, track_name, artist_name, album_name, + album_cover_url, duration_ms, popularity, track_data_json + FROM seasonal_tracks + WHERE season_key = ? AND source = ? + AND spotify_track_id IN ({placeholders}) + """, + (season_key, source, *track_ids), + ) + rows = cursor.fetchall() + + by_id = {} + for r in rows: + if hasattr(r, 'keys'): + r = dict(r) + else: + r = dict(zip( + ('spotify_track_id', 'track_name', 'artist_name', 'album_name', + 'album_cover_url', 'duration_ms', 'popularity', 'track_data_json'), + r, + strict=False, + )) + td = r.get('track_data_json') + if isinstance(td, str): + try: + td = json.loads(td) + except (ValueError, TypeError): + td = None + r['track_data_json'] = td + r['source'] = source + by_id[r['spotify_track_id']] = r + + # Preserve curated order. + return [ + Track.from_dict(by_id[tid]) + for tid in track_ids + if tid in by_id + ] + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + if not variant: + raise ValueError('Seasonal Mix requires a season variant') + seasonal_service = _resolve_seasonal_service(deps) + db = _resolve_database(deps) + source = _resolve_active_source(deps) + track_ids = seasonal_service.get_curated_seasonal_playlist(variant, source=source) or [] + tracks = _hydrate_seasonal_tracks(db, variant, source, track_ids) + return tracks[:config.limit] + + +def variant_resolver(deps: Any) -> List[str]: + """Return every season key from SEASONAL_CONFIG.""" + try: + from core.seasonal_discovery import SEASONAL_CONFIG + except Exception: + return [] + return list(SEASONAL_CONFIG.keys()) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Seasonal — {variant}', + description='Holiday / season-themed picks. One playlist per season; user enables which to track.', + default_config=PlaylistConfig(limit=50, max_per_album=2, max_per_artist=3), + generator=generate, + variant_resolver=variant_resolver, + requires_variant=True, + tags=['curated', 'seasonal'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/generators/time_machine.py b/core/personalized/generators/time_machine.py new file mode 100644 index 00000000..b139be7f --- /dev/null +++ b/core/personalized/generators/time_machine.py @@ -0,0 +1,66 @@ +"""Time Machine generator — by-decade discovery picks. + +Variant = decade label like ``'1980s'`` / ``'1990s'`` / ``'2000s'``. +The variant resolver returns the standard decade set; users see one +playlist per decade (each independently configurable / refreshable). +""" + +from __future__ import annotations + +from typing import Any, List + +from core.personalized.generators._common import coerce_tracks, get_service +from core.personalized.specs import PlaylistKindSpec, get_registry +from core.personalized.types import PlaylistConfig, Track + + +KIND = 'time_machine' + + +# Standard decades the UI exposes. Adjust here when adding eras. +_DEFAULT_DECADES = ('1960s', '1970s', '1980s', '1990s', '2000s', '2010s', '2020s') + + +def _decade_to_year(variant: str) -> int: + """'1980s' -> 1980. Tolerates ' 1980 ', '1980'. + + Raises ValueError for anything that doesn't look like a decade + label so the manager surfaces a clear error instead of generating + garbage.""" + cleaned = (variant or '').strip().rstrip('sS') + try: + year = int(cleaned) + except ValueError as exc: + raise ValueError(f"Time Machine variant {variant!r} not a decade label") from exc + if year < 1900 or year > 2100: + raise ValueError(f"Time Machine variant {variant!r} out of range") + return year + + +def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]: + service = get_service(deps) + decade_year = _decade_to_year(variant) + rows = service.get_decade_playlist(decade=decade_year, limit=config.limit) + return coerce_tracks(rows) + + +def variant_resolver(deps: Any) -> List[str]: + """Return the standard decade set. Future enhancement: filter to + decades that actually have data in the discovery pool.""" + return list(_DEFAULT_DECADES) + + +SPEC = PlaylistKindSpec( + kind=KIND, + name_template='Time Machine — {variant}', + description='Tracks from a specific decade. One playlist per decade.', + default_config=PlaylistConfig(limit=100, max_per_album=3, max_per_artist=5), + generator=generate, + variant_resolver=variant_resolver, + requires_variant=True, + tags=['time'], +) + + +if get_registry().get(KIND) is None: + get_registry().register(SPEC) diff --git a/core/personalized/manager.py b/core/personalized/manager.py new file mode 100644 index 00000000..2e936e42 --- /dev/null +++ b/core/personalized/manager.py @@ -0,0 +1,478 @@ +"""Storage layer + lifecycle for personalized playlists. + +The manager is the ONLY place that touches the +``personalized_playlists`` / ``personalized_playlist_tracks`` / +``personalized_track_history`` tables. Generators (in +``core/personalized/generators/``) produce track lists; the manager +persists, refreshes, and serves them. + +Key invariants: + +- ``(profile_id, kind, variant)`` uniquely identifies a playlist. + Variant '' (empty string) means singleton — used for kinds like + ``hidden_gems`` that don't have multiple instances per profile. +- A playlist row is auto-created on first access (``ensure_playlist``) + using the kind's default config. +- Track lists are atomically replaced on refresh — never partial- + mutated. Either the new snapshot lands fully or the old one + remains. +- Refresh failures get logged on the row + (``last_generation_error``) so the UI can surface them without + losing the previous good snapshot. +- Staleness history is append-only and queried by the + ``exclude_recent_days`` config option (handled by individual + generators when they want to honor it). +""" + +from __future__ import annotations + +import json +import time +from dataclasses import asdict +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from utils.logging_config import get_logger + +from core.personalized.specs import PlaylistKindRegistry, get_registry +from core.personalized.types import PlaylistConfig, PlaylistRecord, Track + +logger = get_logger("personalized.manager") + + +class PersonalizedPlaylistManager: + """Owns persistence + refresh lifecycle for personalized playlists.""" + + def __init__(self, database, deps: Any, registry: Optional[PlaylistKindRegistry] = None): + """ + Args: + database: MusicDatabase singleton (exposes ``_get_connection``). + deps: Opaque object passed through to each generator + callable. Whatever a generator needs (the legacy + ``PersonalizedPlaylistsService`` instance, the + ``config_manager``, a metadata client) goes in here. + Manager doesn't inspect it. + registry: optional PlaylistKindRegistry override (for tests). + """ + self.database = database + self.deps = deps + self.registry = registry or get_registry() + + # ─── playlist row lifecycle ────────────────────────────────────── + + def ensure_playlist(self, kind: str, variant: str = '', profile_id: int = 1) -> PlaylistRecord: + """Return the playlist row for ``(profile, kind, variant)``, + creating it from the kind's default config if it doesn't exist.""" + spec = self.registry.get(kind) + if spec is None: + raise ValueError(f"Unknown playlist kind: {kind!r}") + if spec.requires_variant and not variant: + raise ValueError(f"Kind {kind!r} requires a variant") + + existing = self._fetch_playlist_row(kind, variant, profile_id) + if existing is not None: + return existing + + # Insert new row using the kind's defaults. + config = spec.default_config + name = spec.display_name(variant) + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + INSERT INTO personalized_playlists + (profile_id, kind, variant, name, config_json, + track_count, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (profile_id, kind, variant, name, json.dumps(config.to_json_dict())), + ) + conn.commit() + return self._fetch_playlist_row(kind, variant, profile_id) # type: ignore[return-value] + + def get_playlist(self, kind: str, variant: str = '', profile_id: int = 1) -> Optional[PlaylistRecord]: + """Return the playlist row if it exists. Does NOT auto-create.""" + return self._fetch_playlist_row(kind, variant, profile_id) + + def list_playlists(self, profile_id: int = 1) -> List[PlaylistRecord]: + """List every persisted playlist for a profile, newest-first.""" + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT id, profile_id, kind, variant, name, config_json, + track_count, last_generated_at, last_synced_at, + last_generation_source, last_generation_error, + is_stale + FROM personalized_playlists + WHERE profile_id = ? + ORDER BY COALESCE(last_generated_at, created_at) DESC + """, + (profile_id,), + ) + rows = cursor.fetchall() + return [self._row_to_record(r) for r in rows] + + def update_config(self, kind: str, variant: str, profile_id: int, overrides: Dict[str, Any]) -> PlaylistRecord: + """Patch the per-playlist config with the provided overrides.""" + record = self.ensure_playlist(kind, variant, profile_id) + merged = record.config.merged(overrides) + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE personalized_playlists + SET config_json = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (json.dumps(merged.to_json_dict()), record.id), + ) + conn.commit() + return self._fetch_playlist_row(kind, variant, profile_id) # type: ignore[return-value] + + # ─── refresh / generation ───────────────────────────────────────── + + def refresh_playlist( + self, + kind: str, + variant: str = '', + profile_id: int = 1, + config_overrides: Optional[Dict[str, Any]] = None, + ) -> PlaylistRecord: + """Run the kind's generator and persist the result as the + playlist's current snapshot. + + Atomic: track list is replaced in a single transaction. On + generator exception, the previous snapshot is preserved and + the error is recorded on the row. + + Args: + kind: registered kind identifier. + variant: e.g. '1980s' for time machine, '' for singletons. + profile_id: profile to refresh under. + config_overrides: optional per-call config tweaks merged on + top of the stored config (e.g. UI lets the user "preview + with limit=100" without persisting that change). + + Returns: + Updated PlaylistRecord with fresh ``track_count`` / + ``last_generated_at`` (or ``last_generation_error`` on + failure). + """ + spec = self.registry.get(kind) + if spec is None: + raise ValueError(f"Unknown playlist kind: {kind!r}") + record = self.ensure_playlist(kind, variant, profile_id) + + config = record.config + if config_overrides: + config = config.merged(config_overrides) + + try: + tracks = spec.generator(self.deps, variant, config) + except Exception as exc: # noqa: BLE001 — record + re-raise after persisting + logger.exception("Generator failed for kind=%s variant=%s: %s", kind, variant, exc) + self._record_generation_failure(record.id, str(exc)) + return self._fetch_playlist_row(kind, variant, profile_id) # type: ignore[return-value] + + # Quality post-filters — applied uniformly to every kind so + # generators stay focused on selection logic, not staleness + # bookkeeping. Filters are config-driven; defaults preserve + # the pre-overhaul behavior (no filtering). + tracks = self._apply_quality_filters(tracks, kind, profile_id, config) + + return self._persist_snapshot(record.id, kind, profile_id, tracks) + + def _apply_quality_filters( + self, + tracks: List[Track], + kind: str, + profile_id: int, + config: PlaylistConfig, + ) -> List[Track]: + """Apply manager-level quality filters to a generator's output. + + Currently: + - **Staleness window** (`config.exclude_recent_days > 0`): drops + any track whose primary id was served by this `kind` for this + `profile_id` in the last N days. Prevents the same track + from showing up across consecutive refreshes — e.g. a daily + Discovery Shuffle that shouldn't replay yesterday's picks. + Tracks without a primary id pass through unchanged (nothing + to dedupe on). + + Returns a new list (never mutates input). When no filter + applies, returns ``tracks`` unchanged.""" + if config.exclude_recent_days <= 0 or not tracks: + return tracks + + recent_set = set(self.recent_track_ids(profile_id, kind, config.exclude_recent_days)) + if not recent_set: + return tracks + + return [t for t in tracks if not t.primary_id() or t.primary_id() not in recent_set] + + # ─── track read ────────────────────────────────────────────────── + + def get_playlist_tracks(self, playlist_id: int) -> List[Track]: + """Return the persisted track list for a playlist row.""" + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT spotify_track_id, itunes_track_id, deezer_track_id, + track_name, artist_name, album_name, album_cover_url, + duration_ms, popularity, track_data_json + FROM personalized_playlist_tracks + WHERE playlist_id = ? + ORDER BY position ASC + """, + (playlist_id,), + ) + rows = cursor.fetchall() + return [self._row_to_track(r) for r in rows] + + # ─── snapshot freshness vs source data ─────────────────────────── + + def mark_kinds_stale(self, kinds: List[str], profile_id: Optional[int] = None) -> int: + """Flag every playlist row matching one of ``kinds`` as stale. + + Called by upstream data refreshers (watchlist scan finishing + / Spotify enrichment worker re-pulling Release Radar / etc) + so pipelines auto-regenerate snapshots before the next sync + instead of pushing stale data to the media server. + + Returns the number of rows touched. When ``profile_id`` is + None, flags rows across every profile. + """ + if not kinds: + return 0 + placeholders = ','.join('?' * len(kinds)) + sql = f"UPDATE personalized_playlists SET is_stale = 1, updated_at = CURRENT_TIMESTAMP WHERE kind IN ({placeholders})" + params: List[Any] = list(kinds) + if profile_id is not None: + sql += " AND profile_id = ?" + params.append(profile_id) + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute(sql, params) + count = cursor.rowcount + conn.commit() + return count + + # ─── staleness history ─────────────────────────────────────────── + + def recent_track_ids(self, profile_id: int, kind: str, days: int) -> List[str]: + """Return track IDs served by ``kind`` for ``profile_id`` in + the last ``days`` days. Generators query this when honoring + the ``exclude_recent_days`` config knob.""" + if days <= 0: + return [] + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT DISTINCT track_id + FROM personalized_track_history + WHERE profile_id = ? + AND kind = ? + AND served_at >= datetime('now', ?) + """, + (profile_id, kind, f'-{int(days)} days'), + ) + return [r[0] for r in cursor.fetchall() if r[0]] + + # ─── internal helpers ──────────────────────────────────────────── + + def _persist_snapshot(self, playlist_id: int, kind: str, profile_id: int, tracks: List[Track]) -> PlaylistRecord: + """Atomic replace of a playlist's track list + history append.""" + now = datetime.now(timezone.utc).isoformat(timespec='seconds') + primary_source = next( + (t.source for t in tracks if t.source), None, + ) + with self.database._get_connection() as conn: + cursor = conn.cursor() + try: + cursor.execute("BEGIN") + cursor.execute( + "DELETE FROM personalized_playlist_tracks WHERE playlist_id = ?", + (playlist_id,), + ) + for position, track in enumerate(tracks): + td = track.track_data_json + if td is not None and not isinstance(td, str): + td = json.dumps(td) + cursor.execute( + """ + INSERT INTO personalized_playlist_tracks + (playlist_id, position, + spotify_track_id, itunes_track_id, deezer_track_id, + track_name, artist_name, album_name, album_cover_url, + duration_ms, popularity, track_data_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + playlist_id, position, + track.spotify_track_id, track.itunes_track_id, track.deezer_track_id, + track.track_name, track.artist_name, track.album_name, track.album_cover_url, + int(track.duration_ms or 0), int(track.popularity or 0), td, + ), + ) + cursor.execute( + """ + UPDATE personalized_playlists + SET track_count = ?, last_generated_at = CURRENT_TIMESTAMP, + last_generation_source = ?, last_generation_error = NULL, + is_stale = 0, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (len(tracks), primary_source, playlist_id), + ) + # History append — only for tracks with a primary ID, + # used by exclude_recent_days filter on next refresh. + for track in tracks: + tid = track.primary_id() + if not tid: + continue + cursor.execute( + """ + INSERT INTO personalized_track_history + (profile_id, kind, track_id, served_at) + VALUES (?, ?, ?, CURRENT_TIMESTAMP) + """, + (profile_id, kind, tid), + ) + conn.commit() + except Exception: + conn.rollback() + raise + + # Re-read the row so the returned record carries the fresh + # last_generated_at timestamp. + record = self._fetch_playlist_row_by_id(playlist_id) + if record is None: + raise RuntimeError(f"playlist row {playlist_id} disappeared mid-refresh") + return record + + def _record_generation_failure(self, playlist_id: int, error_message: str) -> None: + """Stamp the error on the row WITHOUT touching tracks.""" + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE personalized_playlists + SET last_generation_error = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (error_message[:500], playlist_id), + ) + conn.commit() + + def _fetch_playlist_row(self, kind: str, variant: str, profile_id: int) -> Optional[PlaylistRecord]: + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT id, profile_id, kind, variant, name, config_json, + track_count, last_generated_at, last_synced_at, + last_generation_source, last_generation_error, + is_stale + FROM personalized_playlists + WHERE profile_id = ? AND kind = ? AND variant = ? + """, + (profile_id, kind, variant), + ) + row = cursor.fetchone() + return self._row_to_record(row) if row else None + + def _fetch_playlist_row_by_id(self, playlist_id: int) -> Optional[PlaylistRecord]: + with self.database._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + """ + SELECT id, profile_id, kind, variant, name, config_json, + track_count, last_generated_at, last_synced_at, + last_generation_source, last_generation_error, + is_stale + FROM personalized_playlists + WHERE id = ? + """, + (playlist_id,), + ) + row = cursor.fetchone() + return self._row_to_record(row) if row else None + + @staticmethod + def _row_to_record(row: Any) -> PlaylistRecord: + # Tolerate sqlite3.Row + plain tuples. + if hasattr(row, 'keys'): + row = dict(row) + return PlaylistRecord( + id=row['id'], profile_id=row['profile_id'], + kind=row['kind'], variant=row['variant'] or '', + name=row['name'], + config=PlaylistConfig.from_json_dict(_safe_json_loads(row['config_json'])), + track_count=row['track_count'] or 0, + last_generated_at=row.get('last_generated_at'), + last_synced_at=row.get('last_synced_at'), + last_generation_source=row.get('last_generation_source'), + last_generation_error=row.get('last_generation_error'), + is_stale=bool(row.get('is_stale') or 0), + ) + # Tuple form: positional access matches SELECT order above. + return PlaylistRecord( + id=row[0], profile_id=row[1], + kind=row[2], variant=row[3] or '', + name=row[4], + config=PlaylistConfig.from_json_dict(_safe_json_loads(row[5])), + track_count=row[6] or 0, + last_generated_at=row[7], + last_synced_at=row[8], + last_generation_source=row[9], + last_generation_error=row[10], + is_stale=bool(row[11] or 0) if len(row) > 11 else False, + ) + + @staticmethod + def _row_to_track(row: Any) -> Track: + if hasattr(row, 'keys'): + row = dict(row) + return Track( + spotify_track_id=row.get('spotify_track_id'), + itunes_track_id=row.get('itunes_track_id'), + deezer_track_id=row.get('deezer_track_id'), + track_name=row.get('track_name', ''), + artist_name=row.get('artist_name', ''), + album_name=row.get('album_name') or '', + album_cover_url=row.get('album_cover_url'), + duration_ms=int(row.get('duration_ms') or 0), + popularity=int(row.get('popularity') or 0), + track_data_json=_safe_json_loads(row.get('track_data_json')), + ) + return Track( + spotify_track_id=row[0], itunes_track_id=row[1], deezer_track_id=row[2], + track_name=row[3], artist_name=row[4], album_name=row[5] or '', + album_cover_url=row[6], duration_ms=int(row[7] or 0), + popularity=int(row[8] or 0), + track_data_json=_safe_json_loads(row[9]), + ) + + +def _safe_json_loads(value: Any) -> Any: + """Tolerant JSON parse — returns None / dict / passes through + non-string values. Avoids exceptions on bad rows so the manager + never breaks on a single corrupt record.""" + if value is None: + return None + if not isinstance(value, str): + return value + if not value.strip(): + return None + try: + return json.loads(value) + except (ValueError, TypeError): + return None + + +__all__ = ['PersonalizedPlaylistManager'] diff --git a/core/personalized/specs.py b/core/personalized/specs.py new file mode 100644 index 00000000..d1367d9c --- /dev/null +++ b/core/personalized/specs.py @@ -0,0 +1,121 @@ +"""Per-kind specifications for the personalized-playlist subsystem. + +A ``PlaylistKindSpec`` declares everything the manager needs to know +about one playlist type: + +- The kind identifier (stable string used in URLs / configs / DB). +- Human-readable display name template (with optional ``{variant}`` + substitution). +- Whether the kind supports / requires variants and what valid + variants look like. +- The default user-tweakable config for this kind. +- A generator callable that produces a fresh track list given + ``(deps, variant, config)``. + +Generators live in ``core/personalized/generators/`` (added in +later commits as each kind is migrated). For commit 1 the registry +ships empty — schema + manager land first; generators arrive +incrementally with their per-kind tests. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional + +from core.personalized.types import PlaylistConfig, Track + + +# Generator callable signature. +# Args: +# deps: opaque object the generator may need (DB, service handle, +# config_manager). Each generator declares what it pulls. +# variant: e.g. '1980s' for time machine, 'halloween' for seasonal. +# Empty string '' for singleton kinds. +# config: PlaylistConfig with the user's per-playlist overrides +# merged onto the kind's defaults. +# Returns: +# List[Track] — the fresh snapshot to persist as the playlist's +# current track list. +GeneratorFn = Callable[[Any, str, PlaylistConfig], List[Track]] + + +# Variant resolver: returns the list of currently-valid variant +# identifiers for a kind that supports multiple instances. Used by +# the manager when auto-creating playlist rows for newly available +# variants (e.g. a new decade, a new season). Singletons return ['']. +VariantResolver = Callable[[Any], List[str]] + + +@dataclass +class PlaylistKindSpec: + """Declaration of one playlist kind. + + See module docstring for the contract. + """ + + kind: str + name_template: str # e.g. 'Time Machine — {variant}', 'Hidden Gems' + description: str + default_config: PlaylistConfig + generator: GeneratorFn + variant_resolver: Optional[VariantResolver] = None + requires_variant: bool = False + # Tags for UI grouping ('curated' / 'discovery' / 'time' / 'genre'). + tags: List[str] = field(default_factory=list) + + def display_name(self, variant: str) -> str: + """Render the human-readable playlist name for a given variant.""" + if not variant: + return self.name_template.replace('{variant}', '').strip(' —-') + return self.name_template.format(variant=variant) + + +class PlaylistKindRegistry: + """Module-level registry of every kind the manager knows about. + + Populated at import time as each generator module is loaded. The + manager queries the registry at runtime to dispatch refresh + requests, list available kinds for the UI, and resolve variants. + """ + + def __init__(self) -> None: + self._kinds: Dict[str, PlaylistKindSpec] = {} + + def register(self, spec: PlaylistKindSpec) -> None: + if spec.kind in self._kinds: + raise ValueError(f"Kind {spec.kind!r} already registered") + self._kinds[spec.kind] = spec + + def get(self, kind: str) -> Optional[PlaylistKindSpec]: + return self._kinds.get(kind) + + def all(self) -> List[PlaylistKindSpec]: + return list(self._kinds.values()) + + def kinds(self) -> List[str]: + return list(self._kinds.keys()) + + def reset_for_tests(self) -> None: + """Drop every registration. Tests only — production runs + register at module import and never reset.""" + self._kinds.clear() + + +# Module-level singleton. Generators register against this on import. +_registry = PlaylistKindRegistry() + + +def get_registry() -> PlaylistKindRegistry: + """Public accessor for the module-level registry. Tests can reset + via ``get_registry().reset_for_tests()``.""" + return _registry + + +__all__ = [ + 'PlaylistKindSpec', + 'PlaylistKindRegistry', + 'GeneratorFn', + 'VariantResolver', + 'get_registry', +] diff --git a/core/personalized/types.py b/core/personalized/types.py new file mode 100644 index 00000000..ef0dccae --- /dev/null +++ b/core/personalized/types.py @@ -0,0 +1,181 @@ +"""Shared dataclasses for the personalized-playlist subsystem. + +These are pure data containers — no business logic, no IO. The +manager + specs + generators all speak in these types so the seam +between them stays mechanical. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class Track: + """A track in a personalized playlist. + + Mirrors the shape returned by + ``PersonalizedPlaylistsService._build_track_dict`` so the legacy + generators can be wrapped without translating fields. Always at + least one of the source IDs is populated; ``track_data_json`` is + the full enriched track object when available (used by sync / + download paths that need richer metadata than just the ID).""" + + track_name: str + artist_name: str + album_name: str = '' + spotify_track_id: Optional[str] = None + itunes_track_id: Optional[str] = None + deezer_track_id: Optional[str] = None + album_cover_url: Optional[str] = None + duration_ms: int = 0 + popularity: int = 0 + track_data_json: Optional[Any] = None # dict OR JSON string OR None + source: Optional[str] = None + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> 'Track': + """Coerce a legacy generator's output dict into a Track. + + Tolerates the ``_artist_genres_raw`` / ``_release_date`` extra- + column passthroughs by ignoring them — this dataclass only + carries the storage-layer fields.""" + return cls( + track_name=d.get('track_name', 'Unknown'), + artist_name=d.get('artist_name', 'Unknown'), + album_name=d.get('album_name', '') or '', + spotify_track_id=d.get('spotify_track_id'), + itunes_track_id=d.get('itunes_track_id'), + deezer_track_id=d.get('deezer_track_id'), + album_cover_url=d.get('album_cover_url'), + duration_ms=int(d.get('duration_ms') or 0), + popularity=int(d.get('popularity') or 0), + track_data_json=d.get('track_data_json'), + source=d.get('source'), + ) + + def primary_id(self) -> Optional[str]: + """Return the first non-empty source ID. Used as the staleness- + history key + the dedupe key when persisting tracks.""" + return ( + self.spotify_track_id + or self.itunes_track_id + or self.deezer_track_id + or None + ) + + +@dataclass +class PlaylistConfig: + """User-tweakable knobs per playlist instance. + + Stored as JSON in `personalized_playlists.config_json`. Defaults + come from the kind's spec; user overrides override per-playlist. + + Fields: + - limit: target number of tracks + - max_per_album / max_per_artist: diversity caps + - popularity_min / popularity_max: filter bounds (None = ignore) + - exclude_recent_days: avoid tracks served by this kind in the + last N days (0 = no exclusion) + - recency_days: only include tracks released in the last N days + (None = all-time) + - seed: optional deterministic seed for randomization (None = + use system random; same seed + same pool = same output) + - extra: free-form per-kind extension dict (e.g. seasonal mix + stores ``selected_seasons``, time machine stores + ``selected_decades``, genre stores ``selected_genres``). + """ + + limit: int = 50 + max_per_album: int = 2 + max_per_artist: int = 3 + popularity_min: Optional[int] = None + popularity_max: Optional[int] = None + exclude_recent_days: int = 0 + recency_days: Optional[int] = None + seed: Optional[int] = None + extra: Dict[str, Any] = field(default_factory=dict) + + def to_json_dict(self) -> Dict[str, Any]: + """Serialise to a JSON-safe dict for storage.""" + return { + 'limit': self.limit, + 'max_per_album': self.max_per_album, + 'max_per_artist': self.max_per_artist, + 'popularity_min': self.popularity_min, + 'popularity_max': self.popularity_max, + 'exclude_recent_days': self.exclude_recent_days, + 'recency_days': self.recency_days, + 'seed': self.seed, + 'extra': dict(self.extra), + } + + @classmethod + def from_json_dict(cls, d: Optional[Dict[str, Any]]) -> 'PlaylistConfig': + """Reconstruct from a stored JSON dict. Missing fields fall + back to defaults so old rows + new code stay compatible.""" + if not isinstance(d, dict): + return cls() + return cls( + limit=int(d.get('limit', 50)), + max_per_album=int(d.get('max_per_album', 2)), + max_per_artist=int(d.get('max_per_artist', 3)), + popularity_min=d.get('popularity_min'), + popularity_max=d.get('popularity_max'), + exclude_recent_days=int(d.get('exclude_recent_days', 0)), + recency_days=d.get('recency_days'), + seed=d.get('seed'), + extra=dict(d.get('extra') or {}), + ) + + def merged(self, overrides: Dict[str, Any]) -> 'PlaylistConfig': + """Return a new PlaylistConfig with `overrides` merged in. + + Used when a user PATCHes their per-playlist config — apply + only the fields they sent, leave the rest at their stored + values.""" + base = self.to_json_dict() + for key, value in (overrides or {}).items(): + if key == 'extra' and isinstance(value, dict): + base['extra'] = {**base.get('extra', {}), **value} + elif key in base: + base[key] = value + return PlaylistConfig.from_json_dict(base) + + +@dataclass +class PlaylistRecord: + """One row of `personalized_playlists` plus its track count. + + The live track list is fetched separately via + ``PersonalizedPlaylistManager.get_playlist_tracks(playlist_id)`` + so list / detail responses can stay cheap when the caller only + needs metadata. + + ``is_stale`` flips to True when the underlying source data + changes (e.g. watchlist scan updates the discovery pool) and is + cleared on the next successful refresh. Pipelines auto-refresh + stale snapshots before syncing so the server playlist always + reflects the latest source data.""" + + id: int + profile_id: int + kind: str + variant: str + name: str + config: PlaylistConfig + track_count: int + last_generated_at: Optional[str] + last_synced_at: Optional[str] + last_generation_source: Optional[str] + last_generation_error: Optional[str] + is_stale: bool = False + + +__all__ = [ + 'Track', + 'PlaylistConfig', + 'PlaylistRecord', +] diff --git a/core/personalized_playlists.py b/core/personalized_playlists.py index 64882b97..602520ab 100644 --- a/core/personalized_playlists.py +++ b/core/personalized_playlists.py @@ -779,19 +779,21 @@ class PersonalizedPlaylistsService: } def _get_library_tracks_by_category(self, category: str, limit: int) -> List[Dict]: - """ - Get tracks from library matching genre or artist + """Library tracks intentionally excluded from daily mixes. - NOTE: This requires library tracks to have Spotify metadata which may not be available. - Returns empty list if schema incompatible. - """ - try: - logger.warning("Library tracks by category requires Spotify-linked library - returning empty") - return [] + The legacy ambition was 50% library + 50% discovery, but the + ``tracks`` table doesn't carry source IDs (no + ``spotify_track_id`` / ``itunes_track_id`` / ``deezer_track_id`` + column) — so library rows can't flow through the same + sync / wishlist pipeline that discovery tracks do. Returning + them here would produce un-syncable, un-downloadable phantom + entries. - except Exception as e: - logger.error(f"Error getting library tracks by category: {e}") - return [] + Returns ``[]`` so callers compose with ``_get_discovery_tracks_by_category`` + for a discovery-only mix. A future PR can backfill source IDs + into library rows and lift this restriction. + """ + return [] def _get_discovery_tracks_by_category(self, category: str, limit: int) -> List[Dict]: """Get tracks from discovery pool matching genre or artist""" diff --git a/core/reorganize_queue.py b/core/reorganize_queue.py index 6b50afb0..a86a638c 100644 --- a/core/reorganize_queue.py +++ b/core/reorganize_queue.py @@ -66,6 +66,10 @@ class QueueItem: artist_name: str # captured at enqueue time for UI display source: Optional[str] # the user's per-modal pick (None = auto) enqueued_at: float + # 'api' (default) = query metadata source per album_data IDs. + # 'tags' = read each file's embedded tags as the source + # of truth (issue #592). Zero API calls. + metadata_source: str = 'api' status: str = 'queued' # queued | running | done | failed | cancelled started_at: Optional[float] = None finished_at: Optional[float] = None @@ -91,6 +95,7 @@ class QueueItem: 'artist_id': self.artist_id, 'artist_name': self.artist_name, 'source': self.source, + 'metadata_source': self.metadata_source, 'enqueued_at': self.enqueued_at, 'started_at': self.started_at, 'finished_at': self.finished_at, @@ -155,6 +160,7 @@ class ReorganizeQueue: artist_id: Optional[str], artist_name: str, source: Optional[str] = None, + metadata_source: str = 'api', ) -> dict: """Add an album to the queue. Returns a result dict: @@ -183,6 +189,7 @@ class ReorganizeQueue: artist_name=artist_name, source=source, enqueued_at=time.time(), + metadata_source=metadata_source or 'api', ) self._items.append(item) position = sum(1 for i in self._items if i.status == 'queued') @@ -190,7 +197,8 @@ class ReorganizeQueue: self._cond.notify_all() logger.info( f"[Queue] Enqueued '{album_title}' (album_id={album_id}, " - f"queue_id={item.queue_id}, position={position}, source={source or 'auto'})" + f"queue_id={item.queue_id}, position={position}, " + f"source={source or 'auto'}, metadata={item.metadata_source})" ) return { 'queued': True, @@ -240,6 +248,7 @@ class ReorganizeQueue: artist_name=raw.get('artist_name') or 'Unknown Artist', source=raw.get('source'), enqueued_at=time.time(), + metadata_source=raw.get('metadata_source') or 'api', ) self._items.append(item) enqueued += 1 diff --git a/core/reorganize_runner.py b/core/reorganize_runner.py index 8ef7817b..1b78e932 100644 --- a/core/reorganize_runner.py +++ b/core/reorganize_runner.py @@ -118,6 +118,7 @@ def build_runner( primary_source=item.source, strict_source=bool(item.source), stop_check=is_shutting_down_fn, + metadata_source=getattr(item, 'metadata_source', 'api') or 'api', ) return runner diff --git a/core/search/orchestrator.py b/core/search/orchestrator.py index da934759..a1961424 100644 --- a/core/search/orchestrator.py +++ b/core/search/orchestrator.py @@ -31,7 +31,7 @@ from . import sources logger = logging.getLogger(__name__) VALID_SOURCES = ( - 'spotify', 'itunes', 'deezer', 'discogs', 'hydrabase', 'musicbrainz', + 'spotify', 'itunes', 'deezer', 'discogs', 'hydrabase', 'musicbrainz', 'amazon', ) VALID_STREAM_SOURCES = VALID_SOURCES + ('youtube_videos',) @@ -88,6 +88,13 @@ def resolve_client(source_name: str, deps: SearchDeps) -> tuple[Any, bool]: except Exception as e: logger.warning(f"MusicBrainz search client init failed: {e}") return None, False + if source_name == 'amazon': + try: + from core.metadata.registry import get_amazon_client + return get_amazon_client(), True + except Exception as e: + logger.warning(f"Amazon Music client init failed: {e}") + return None, False return None, False @@ -183,6 +190,8 @@ def _alternate_sources(primary_source: str, deps: SearchDeps) -> list[str]: alts.append('discogs') if primary_source != 'hydrabase' and hydrabase_available: alts.append('hydrabase') + if primary_source != 'amazon': + alts.append('amazon') # always available (T2Tunes, no auth) alts.append('youtube_videos') # always available (yt-dlp, no auth) alts.append('musicbrainz') # always available (public API) return alts diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index c90e95bd..5bf82809 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -24,6 +24,19 @@ from core.wishlist_service import get_wishlist_service from core.matching_engine import MusicMatchingEngine from utils.logging_config import get_logger + +def _mark_personalized_kinds_stale(database, kinds, profile_id=1): + """Module-level helper so the inline call sites stay tiny. + + Constructs a PersonalizedPlaylistManager with the minimal deps + needed for `mark_kinds_stale` (database access only — no generator + dispatch required) and flips the is_stale flag for matching rows. + Best-effort: any exception is swallowed by the caller's try/except + since stale-flagging is non-critical for the scan itself.""" + from core.personalized.manager import PersonalizedPlaylistManager + mgr = PersonalizedPlaylistManager(database, deps=None) + return mgr.mark_kinds_stale(list(kinds), profile_id=profile_id) + logger = get_logger("watchlist_scanner") # Rate limiting constants for watchlist operations @@ -2980,6 +2993,22 @@ class WatchlistScanner: self.database.update_discovery_pool_timestamp(track_count=final_count, profile_id=profile_id) logger.info(f"Discovery pool now contains {final_count} total tracks (built over time)") + # Mark every personalized-playlist kind that draws from the + # discovery pool as stale so the playlist pipeline auto- + # regenerates snapshots on its next run. Without this the + # server playlists stay frozen even though the source pool + # just got fresh tracks. Best-effort — pool refresh succeeds + # even if the manager isn't wired (no personalized tables). + try: + _mark_personalized_kinds_stale( + self.database, + kinds=['hidden_gems', 'discovery_shuffle', 'popular_picks', + 'time_machine', 'genre_playlist', 'daily_mix'], + profile_id=profile_id, + ) + except Exception as e: # noqa: BLE001 — never abort scan for staleness flag + logger.debug("Failed to mark personalized kinds stale: %s", e) + # Cache recent albums for discovery page logger.info("Caching recent albums for discovery page...") if progress_callback: @@ -3660,6 +3689,14 @@ class WatchlistScanner: playlist_key = f'release_radar_{source}' self.database.save_curated_playlist(playlist_key, release_radar_tracks, profile_id=profile_id) logger.info(f"Release Radar ({source}) curated: {len(release_radar_tracks)} tracks") + # Flag personalized Fresh Tape snapshot as stale so the + # pipeline auto-regenerates it on the next run. + try: + _mark_personalized_kinds_stale( + self.database, kinds=['fresh_tape'], profile_id=profile_id, + ) + except Exception as e: # noqa: BLE001 + logger.debug("Fresh Tape stale-flag failed: %s", e) # 2. Curate Discovery Weekly - 50 tracks from discovery pool logger.info(f"Curating Discovery Weekly for {source}...") @@ -3735,6 +3772,12 @@ class WatchlistScanner: playlist_key = f'discovery_weekly_{source}' self.database.save_curated_playlist(playlist_key, discovery_weekly_tracks, profile_id=profile_id) logger.info(f"Discovery Weekly ({source}) curated: {len(discovery_weekly_tracks)} tracks") + try: + _mark_personalized_kinds_stale( + self.database, kinds=['archives'], profile_id=profile_id, + ) + except Exception as e: # noqa: BLE001 + logger.debug("Archives stale-flag failed: %s", e) # 3. "Because You Listen To" — personalized sections based on top played artists if profile['has_data']: diff --git a/database/music_database.py b/database/music_database.py index e6a1c35a..a89970c3 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -407,6 +407,9 @@ class MusicDatabase: # Add Discogs enrichment columns (migration) self._add_discogs_columns(cursor) + # Add Amazon artist ID column (migration) + self._add_amazon_columns(cursor) + # Backfill match_status for rows that already have an external ID but # NULL status. Prevents enrichment workers from re-processing these # rows forever. Must run AFTER all *_match_status columns have been @@ -752,13 +755,21 @@ class MusicDatabase: ) """) + # Personalized-playlists subsystem schema (Group A + Group B + # unified storage). Idempotent — safe on every startup. + try: + from database.personalized_schema import ensure_personalized_schema + ensure_personalized_schema(conn) + except Exception as ps_err: + logger.error(f"Personalized-playlist schema init failed: {ps_err}") + conn.commit() logger.info("Database initialized successfully") except Exception as e: logger.error(f"Error initializing database: {e}") raise - + def _add_mirrored_playlist_explored_column(self, cursor): """Add explored_at column to mirrored_playlists to persist explore badge.""" try: @@ -1623,6 +1634,10 @@ class MusicDatabase: cursor.execute("ALTER TABLE watchlist_artists ADD COLUMN discogs_artist_id TEXT") logger.info("Added discogs_artist_id column to watchlist_artists table for cross-provider support") + if 'amazon_artist_id' not in columns: + cursor.execute("ALTER TABLE watchlist_artists ADD COLUMN amazon_artist_id TEXT") + logger.info("Added amazon_artist_id column to watchlist_artists table for Amazon Music support") + except Exception as e: logger.error(f"Error adding itunes_artist_id column to watchlist_artists: {e}") # Don't raise - this is a migration, database can still function @@ -2027,6 +2042,55 @@ class MusicDatabase: except Exception as e: logger.error(f"Error adding Discogs columns: {e}") + def _add_amazon_columns(self, cursor): + """Add Amazon enrichment tracking columns to artists, albums, and tracks.""" + try: + # --- Artists --- + cursor.execute("PRAGMA table_info(artists)") + artists_columns = [column[1] for column in cursor.fetchall()] + + if 'amazon_id' not in artists_columns: + cursor.execute("ALTER TABLE artists ADD COLUMN amazon_id TEXT") + if 'amazon_match_status' not in artists_columns: + cursor.execute("ALTER TABLE artists ADD COLUMN amazon_match_status TEXT") + if 'amazon_last_attempted' not in artists_columns: + cursor.execute("ALTER TABLE artists ADD COLUMN amazon_last_attempted TIMESTAMP") + + cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_amazon_id ON artists (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_amazon_status ON artists (amazon_match_status)") + + # --- Albums --- + cursor.execute("PRAGMA table_info(albums)") + albums_columns = [column[1] for column in cursor.fetchall()] + + if 'amazon_id' not in albums_columns: + cursor.execute("ALTER TABLE albums ADD COLUMN amazon_id TEXT") + if 'amazon_match_status' not in albums_columns: + cursor.execute("ALTER TABLE albums ADD COLUMN amazon_match_status TEXT") + if 'amazon_last_attempted' not in albums_columns: + cursor.execute("ALTER TABLE albums ADD COLUMN amazon_last_attempted TIMESTAMP") + + cursor.execute("CREATE INDEX IF NOT EXISTS idx_albums_amazon_id ON albums (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_albums_amazon_status ON albums (amazon_match_status)") + + # --- Tracks --- + cursor.execute("PRAGMA table_info(tracks)") + tracks_columns = [column[1] for column in cursor.fetchall()] + + if 'amazon_id' not in tracks_columns: + cursor.execute("ALTER TABLE tracks ADD COLUMN amazon_id TEXT") + if 'amazon_match_status' not in tracks_columns: + cursor.execute("ALTER TABLE tracks ADD COLUMN amazon_match_status TEXT") + if 'amazon_last_attempted' not in tracks_columns: + cursor.execute("ALTER TABLE tracks ADD COLUMN amazon_last_attempted TIMESTAMP") + + cursor.execute("CREATE INDEX IF NOT EXISTS idx_tracks_amazon_id ON tracks (amazon_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_tracks_amazon_status ON tracks (amazon_match_status)") + + logger.info("Amazon columns added/verified successfully") + except Exception as e: + logger.error(f"Error adding Amazon columns: {e}") + def _backfill_match_status_for_existing_ids(self, cursor): """Set `_match_status = 'matched'` for rows that already have a populated external ID but NULL match_status. @@ -8895,6 +8959,7 @@ class MusicDatabase: a.tidal_id, a.qobuz_id, a.soul_id, + a.amazon_id, a.server_source FROM artists a WHERE {where_clause} @@ -8994,6 +9059,7 @@ class MusicDatabase: 'tidal_id': row['tidal_id'], 'qobuz_id': row['qobuz_id'], 'soul_id': row['soul_id'], + 'amazon_id': row['amazon_id'], 'album_count': counts_map.get(row['id'], (0, 0))[0], 'track_count': counts_map.get(row['id'], (0, 0))[1], 'is_watched': bool(is_watched) @@ -9052,7 +9118,7 @@ class MusicDatabase: id, name, thumb_url, genres, server_source, musicbrainz_id, deezer_id, audiodb_id, discogs_id, spotify_artist_id, itunes_artist_id, lastfm_url, genius_url, - tidal_id, qobuz_id, soul_id, + tidal_id, qobuz_id, soul_id, amazon_id, lastfm_listeners, lastfm_playcount, lastfm_tags, lastfm_bio FROM artists WHERE id = ? @@ -9210,6 +9276,7 @@ class MusicDatabase: 'tidal_id': artist_row['tidal_id'], 'qobuz_id': artist_row['qobuz_id'], 'soul_id': artist_row['soul_id'], + 'amazon_id': artist_row['amazon_id'], 'lastfm_listeners': artist_row['lastfm_listeners'], 'lastfm_playcount': artist_row['lastfm_playcount'], 'lastfm_tags': artist_row['lastfm_tags'], diff --git a/database/personalized_schema.py b/database/personalized_schema.py new file mode 100644 index 00000000..27df8477 --- /dev/null +++ b/database/personalized_schema.py @@ -0,0 +1,159 @@ +"""Schema + migration helpers for the personalized-playlists subsystem. + +Pre-existing state (this PR replaces over time): +- Group A (Fresh Tape / The Archives / Seasonal Mix) lives in + `discovery_curated_playlists` (track_ids only) and + `curated_seasonal_playlists` (track_ids + seasonal_tracks join). + Read paths exist; refresh paths are tied to specific workers. +- Group B (Hidden Gems / Discovery Shuffle / Time Machine / Popular + Picks / Genre / Daily Mixes) is computed on-demand by + `PersonalizedPlaylistsService` — no persistence, every call reruns + the generator with `ORDER BY RANDOM()` so the result rotates. + +Post-overhaul (this module's responsibility): +- ALL personalized playlists land in a unified storage layer with a + stable (profile_id, kind, variant) identity, JSON config per + playlist (limit, diversity caps, popularity / recency filters, + exclude-recent-days, randomization seed), and a persistent track + list that only mutates when the playlist is explicitly refreshed. + +Tables created here: + +- ``personalized_playlists`` — one row per (profile, kind, variant). + Variants disambiguate kinds with multiple instances: + * ``time_machine``: variant = ``'1980s'`` / ``'1990s'`` / ... + * ``seasonal_mix``: variant = ``'halloween'`` / ``'christmas'`` / ... + * ``genre_playlist``: variant = ``'rock'`` / ``'electronic_dance'`` / ... + * ``daily_mix``: variant = ``'1'`` / ``'2'`` / ``'3'`` / ``'4'`` + * Singletons (``hidden_gems``, ``discovery_shuffle``, + ``popular_picks``, ``fresh_tape``, ``archives``): variant = ``''``. + Variant '' (empty) is used instead of NULL so the UNIQUE + constraint behaves predictably (NULL doesn't collide with NULL in + SQLite UNIQUE indexes — would let multiple singleton rows + coexist). + +- ``personalized_playlist_tracks`` — current snapshot per playlist. + Cleared + repopulated on refresh; never partial-mutates. + +- ``personalized_track_history`` — append-only log of which tracks + were served by which (profile, kind) over time. Powers the + ``exclude_recent_days`` config option so generators can avoid + recommending the same track twice in N days. + +The schema is created idempotently — `ensure_personalized_schema` +runs CREATE TABLE IF NOT EXISTS at startup, so existing installs +upgrade silently.""" + +from __future__ import annotations + +from typing import Any + +from utils.logging_config import get_logger + +logger = get_logger("database.personalized_schema") + + +PERSONALIZED_PLAYLISTS_DDL = """ +CREATE TABLE IF NOT EXISTS personalized_playlists ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + profile_id INTEGER NOT NULL DEFAULT 1, + kind TEXT NOT NULL, + variant TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL, + config_json TEXT NOT NULL DEFAULT '{}', + track_count INTEGER NOT NULL DEFAULT 0, + last_generated_at TIMESTAMP, + last_synced_at TIMESTAMP, + last_generation_source TEXT, + last_generation_error TEXT, + is_stale INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE (profile_id, kind, variant) +) +""" + +# Migration for installs that created the table before is_stale existed. +PERSONALIZED_PLAYLISTS_STALE_MIGRATION = """ +ALTER TABLE personalized_playlists ADD COLUMN is_stale INTEGER NOT NULL DEFAULT 0 +""" + +PERSONALIZED_PLAYLIST_TRACKS_DDL = """ +CREATE TABLE IF NOT EXISTS personalized_playlist_tracks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + playlist_id INTEGER NOT NULL, + position INTEGER NOT NULL, + spotify_track_id TEXT, + itunes_track_id TEXT, + deezer_track_id TEXT, + track_name TEXT, + artist_name TEXT, + album_name TEXT, + album_cover_url TEXT, + duration_ms INTEGER, + popularity INTEGER, + track_data_json TEXT, + FOREIGN KEY (playlist_id) REFERENCES personalized_playlists(id) ON DELETE CASCADE, + UNIQUE (playlist_id, position) +) +""" + +PERSONALIZED_PLAYLIST_TRACKS_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_personalized_tracks_playlist + ON personalized_playlist_tracks(playlist_id) +""" + +PERSONALIZED_TRACK_HISTORY_DDL = """ +CREATE TABLE IF NOT EXISTS personalized_track_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + profile_id INTEGER NOT NULL DEFAULT 1, + kind TEXT NOT NULL, + track_id TEXT NOT NULL, + served_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +) +""" + +PERSONALIZED_TRACK_HISTORY_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_personalized_history_lookup + ON personalized_track_history(profile_id, kind, track_id, served_at) +""" + + +def ensure_personalized_schema(connection: Any) -> None: + """Create the personalized-playlist tables + indexes if missing. + + Idempotent. Safe to call on every app startup. Caller is responsible + for committing the connection (we leave that to the caller so this + composes with other schema-init steps in one transaction). + """ + cursor = connection.cursor() + cursor.execute(PERSONALIZED_PLAYLISTS_DDL) + cursor.execute(PERSONALIZED_PLAYLIST_TRACKS_DDL) + cursor.execute(PERSONALIZED_PLAYLIST_TRACKS_INDEX) + cursor.execute(PERSONALIZED_TRACK_HISTORY_DDL) + cursor.execute(PERSONALIZED_TRACK_HISTORY_INDEX) + + # Add is_stale column on installs that created the table before + # this column existed. SQLite has no `ADD COLUMN IF NOT EXISTS` so + # we probe with PRAGMA + tolerate the OperationalError that fires + # when the column is already there. + cursor.execute("PRAGMA table_info(personalized_playlists)") + cols = {row[1] for row in cursor.fetchall()} + if 'is_stale' not in cols: + try: + cursor.execute(PERSONALIZED_PLAYLISTS_STALE_MIGRATION) + logger.info("Added is_stale column to personalized_playlists") + except Exception as e: + logger.debug("is_stale column migration: %s", e) + + logger.debug("Personalized-playlist schema ensured") + + +__all__ = [ + 'ensure_personalized_schema', + 'PERSONALIZED_PLAYLISTS_DDL', + 'PERSONALIZED_PLAYLIST_TRACKS_DDL', + 'PERSONALIZED_PLAYLIST_TRACKS_INDEX', + 'PERSONALIZED_TRACK_HISTORY_DDL', + 'PERSONALIZED_TRACK_HISTORY_INDEX', +] diff --git a/tests/automation/test_automation_blocks.py b/tests/automation/test_automation_blocks.py index 64d95cfa..151725bb 100644 --- a/tests/automation/test_automation_blocks.py +++ b/tests/automation/test_automation_blocks.py @@ -34,7 +34,8 @@ def _shape_check(items, allowed_types): _FIELD_TYPES = { 'number', 'select', 'time', 'multi_select', 'checkbox', 'text', - 'mirrored_playlist_select', 'signal_input', 'script_select', + 'mirrored_playlist_select', 'personalized_playlist_select', + 'signal_input', 'script_select', } diff --git a/tests/automation/test_handler_registration.py b/tests/automation/test_handler_registration.py new file mode 100644 index 00000000..b6660541 --- /dev/null +++ b/tests/automation/test_handler_registration.py @@ -0,0 +1,391 @@ +"""Engine-boundary tests for the automation handler registration. + +Per-handler boundary tests in the sibling test files prove each +handler's body works in isolation. These tests prove the +**registration layer** wires every handler to the right action name, +attaches the right guard, and registers the four progress callbacks +in the slots the engine expects. + +The kettui standard for refactor PRs: don't ship a "behavior +preserved" claim that's only validated at the function boundary. +Wire the seam — engine + register_all + deps — and exercise it. + +These tests use a minimal recording engine that captures every +``register_action_handler`` / ``register_progress_callbacks`` call, +plus a no-op ``add_scan_completion_callback`` on a fake scan +manager. No real AutomationEngine, no real DB, no real Flask app. +""" + +from __future__ import annotations + +import threading +from typing import Any, Dict, List, Tuple + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers import register_all + + +# Every action name `register_all` is expected to register. Drift +# (rename / new handler / removed handler) fails this test +# immediately so refactor PRs can't quietly drop a handler. +EXPECTED_ACTION_NAMES = frozenset({ + 'process_wishlist', + 'scan_watchlist', + 'scan_library', + 'refresh_mirrored', + 'sync_playlist', + 'discover_playlist', + 'playlist_pipeline', + 'personalized_pipeline', + 'start_database_update', + 'deep_scan_library', + 'run_duplicate_cleaner', + 'clear_quarantine', + 'cleanup_wishlist', + 'update_discovery_pool', + 'start_quality_scan', + 'backup_database', + 'refresh_beatport_cache', + 'clean_search_history', + 'clean_completed_downloads', + 'full_cleanup', + 'run_script', + 'search_and_download', +}) + +# Action names that MUST register a guard (duplicate-run prevention). +EXPECTED_GUARDED_ACTIONS = frozenset({ + 'process_wishlist', + 'scan_watchlist', + 'scan_library', + 'playlist_pipeline', + 'personalized_pipeline', + 'start_database_update', + 'deep_scan_library', + 'run_duplicate_cleaner', + 'start_quality_scan', +}) + + +class _RecordingEngine: + """Minimal AutomationEngine stand-in. Captures everything + register_all does so tests can assert on it.""" + + def __init__(self): + self.action_handlers: Dict[str, Dict[str, Any]] = {} + self.progress_callbacks: Tuple = () + self.emits: List[Tuple[str, dict]] = [] + + def register_action_handler(self, action_type, handler_fn, guard_fn=None): + self.action_handlers[action_type] = {'handler': handler_fn, 'guard': guard_fn} + + def register_progress_callbacks(self, init_fn, finish_fn, update_fn=None, history_fn=None): + self.progress_callbacks = (init_fn, finish_fn, update_fn, history_fn) + + def emit(self, event_name, payload): + self.emits.append((event_name, payload)) + + +class _RecordingScanMgr: + _current_server_type = 'plex' + + def __init__(self): + self.callbacks: List = [] + + def add_scan_completion_callback(self, cb): + self.callbacks.append(cb) + + +def _build_deps(engine, scan_mgr=None) -> AutomationDeps: + class _Logger: + def debug(self, *a, **k): pass + def info(self, *a, **k): pass + def warning(self, *a, **k): pass + def error(self, *a, **k): pass + + class _Cfg: + def get(self, key, default=None): return default + def get_active_media_server(self): return 'plex' + + return AutomationDeps( + engine=engine, + state=AutomationState(), + config_manager=_Cfg(), + update_progress=lambda *a, **k: None, + logger=_Logger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=scan_mgr, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + + +# ─── action handler registration ───────────────────────────────────── + + +class TestActionHandlerRegistration: + def test_every_expected_action_name_registered(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + registered = set(engine.action_handlers.keys()) + missing = EXPECTED_ACTION_NAMES - registered + extra = registered - EXPECTED_ACTION_NAMES + assert not missing, f"register_all dropped: {missing}" + assert not extra, f"register_all added unexpected: {extra}" + + def test_guarded_actions_have_a_guard(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + for name in EXPECTED_GUARDED_ACTIONS: + assert engine.action_handlers[name]['guard'] is not None, ( + f"action {name!r} expected to register a guard but didn't" + ) + + def test_unguarded_actions_have_no_guard(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + unguarded = EXPECTED_ACTION_NAMES - EXPECTED_GUARDED_ACTIONS + for name in unguarded: + assert engine.action_handlers[name]['guard'] is None, ( + f"action {name!r} unexpectedly registered a guard" + ) + + def test_every_handler_callable(self): + # Every registered handler must be callable with a config dict. + engine = _RecordingEngine() + register_all(_build_deps(engine)) + for name, entry in engine.action_handlers.items(): + handler = entry['handler'] + assert callable(handler), f"{name} handler is not callable" + + def test_every_guard_callable_returns_bool(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + for name, entry in engine.action_handlers.items(): + guard = entry['guard'] + if guard is None: + continue + value = guard() + assert isinstance(value, bool), ( + f"{name} guard returned non-bool: {type(value).__name__}" + ) + + +# ─── progress callback registration ────────────────────────────────── + + +class TestProgressCallbackRegistration: + def test_all_four_callbacks_registered(self): + engine = _RecordingEngine() + register_all(_build_deps(engine)) + init_fn, finish_fn, update_fn, history_fn = engine.progress_callbacks + assert callable(init_fn) + assert callable(finish_fn) + assert callable(update_fn) + assert callable(history_fn) + + def test_progress_init_callback_invocable(self): + # Engine signature: init_fn(aid, name, action_type) + engine = _RecordingEngine() + captured: List[Tuple] = [] + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'init_automation_progress': lambda aid, name, at: captured.append((aid, name, at)), + }) + register_all(deps) + init_fn, _, _, _ = engine.progress_callbacks + init_fn('auto-1', 'My Auto', 'wishlist') + assert captured == [('auto-1', 'My Auto', 'wishlist')] + + def test_progress_finish_callback_invocable(self): + # Engine signature: finish_fn(aid, result) + engine = _RecordingEngine() + captured: List[Tuple] = [] + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'update_progress': lambda aid, **kw: captured.append((aid, kw)), + }) + register_all(deps) + _, finish_fn, _, _ = engine.progress_callbacks + # Non-_manages_own_progress result triggers update_progress emit. + finish_fn('auto-1', {'status': 'completed'}) + assert len(captured) == 1 + assert captured[0][0] == 'auto-1' + assert captured[0][1]['status'] == 'finished' + + def test_progress_finish_skips_self_managed(self): + engine = _RecordingEngine() + captured: List[Tuple] = [] + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'update_progress': lambda aid, **kw: captured.append((aid, kw)), + }) + register_all(deps) + _, finish_fn, _, _ = engine.progress_callbacks + finish_fn('auto-1', {'_manages_own_progress': True, 'status': 'completed'}) + assert captured == [] + + def test_history_callback_invocable_with_db(self): + # Engine signature: history_fn(aid, result) + engine = _RecordingEngine() + captured: List[Tuple] = [] + db_obj = object() + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'get_database': lambda: db_obj, + 'record_progress_history': lambda aid, result, db: captured.append((aid, result, db)), + }) + register_all(deps) + _, _, _, history_fn = engine.progress_callbacks + history_fn('auto-1', {'status': 'completed'}) + assert captured == [('auto-1', {'status': 'completed'}, db_obj)] + + +# ─── library_scan_completed wiring ─────────────────────────────────── + + +class TestLibraryScanCompletedEmitter: + def test_no_scan_manager_safe(self): + # Should not raise when scan manager is absent (test/headless mode). + engine = _RecordingEngine() + register_all(_build_deps(engine, scan_mgr=None)) + # No callbacks captured (no scan manager to register against), + # but engine still has all the handlers. + assert engine.action_handlers + + def test_scan_completion_callback_registered(self): + engine = _RecordingEngine() + scan_mgr = _RecordingScanMgr() + register_all(_build_deps(engine, scan_mgr=scan_mgr)) + assert len(scan_mgr.callbacks) == 1 + # Invoking the callback should fire engine.emit('library_scan_completed', ...). + scan_mgr.callbacks[0]() + assert engine.emits == [('library_scan_completed', {'server_type': 'plex'})] + + +# ─── handler invocation through the engine boundary ───────────────── + + +class TestHandlerInvocation: + """For each registered handler, exercise the lambda the engine + would call. Verifies the deps closure is captured correctly + the + handler returns a result dict. + + Forces every long-running handler down its guard short-circuit + path by pre-setting the relevant `*_state` dicts to ``running`` + (database_update, deep_scan, duplicate_cleaner, quality_scanner) + or by pre-occupying the state flags (scan_library, playlist_pipeline). + Other handlers either return error early (no playlist specified, + no scan manager, etc) or run cleanly against the no-op stub deps. + """ + + def test_every_handler_returns_dict(self): + engine = _RecordingEngine() + # Pre-set state dicts so guarded handlers skip-return cleanly. + running_state = {'status': 'running'} + active_batches = {'b1': {'phase': 'downloading'}} + + # Stub DB that satisfies the handlers reaching for it on + # short paths. cleanup_wishlist calls remove_wishlist_duplicates. + # update_discovery_pool's exception path swallows missing + # scanner. Other handlers either short-circuit or use stub + # callables. + class _StubDB: + def remove_wishlist_duplicates(self, profile_id): return 0 + def get_mirrored_playlists(self): return [] + def get_mirrored_playlist(self, _id): return None + def get_mirrored_playlist_tracks(self, _id): return [] + + # Stub Flask app so refresh_beatport_cache's test_client call + # doesn't crash. Use a minimal context manager. + class _FakeResponse: + status_code = 200 + class _FakeClient: + def __enter__(self): return self + def __exit__(self, *a): return False + def get(self, _path): return _FakeResponse() + class _StubApp: + def test_client(self): return _FakeClient() + + deps = _build_deps(engine) + deps = AutomationDeps(**{ + **{f.name: getattr(deps, f.name) for f in deps.__dataclass_fields__.values()}, + 'get_db_update_state': lambda: running_state, + 'get_duplicate_cleaner_state': lambda: running_state, + 'get_quality_scanner_state': lambda: running_state, + 'get_download_batches': lambda: active_batches, # forces clean_completed_downloads to skip + 'get_database': lambda: _StubDB(), + 'get_app': lambda: _StubApp(), + }) + # Pre-set state flags too so scan_library + playlist_pipeline guards fire. + deps.state.scan_library_automation_id = 'someone-else' + deps.state.pipeline_running = True + + # Patch time.sleep across handler modules so refresh_beatport_cache + # (which sleeps 2s between sections) doesn't extend the test. + import core.automation.handlers.maintenance as maint_mod + original_sleep = maint_mod.time.sleep + maint_mod.time.sleep = lambda _: None + + register_all(deps) + + try: + for name, entry in engine.action_handlers.items(): + handler = entry['handler'] + # Minimal config: trigger the natural error/skip paths + # for handlers that need a playlist_id, query, script_name etc. + result = handler({'_automation_id': 'test', 'all': False}) + assert isinstance(result, dict), ( + f"handler {name!r} returned {type(result).__name__}, expected dict" + ) + assert 'status' in result, ( + f"handler {name!r} returned dict without 'status': {list(result.keys())}" + ) + finally: + maint_mod.time.sleep = original_sleep diff --git a/tests/automation/test_handlers_maintenance.py b/tests/automation/test_handlers_maintenance.py new file mode 100644 index 00000000..a3844b77 --- /dev/null +++ b/tests/automation/test_handlers_maintenance.py @@ -0,0 +1,393 @@ +"""Boundary tests for the maintenance + misc automation handlers +(database_update / deep_scan_library / duplicate_cleaner / +quality_scanner / clear_quarantine / cleanup_wishlist / +update_discovery_pool / backup_database / refresh_beatport_cache / +clean_search_history / clean_completed_downloads / full_cleanup / +run_script / search_and_download). + +Each handler is tested as a pure function via stub deps. The bodies +are mechanical lifts of the closures that used to live in +``web_server._register_automation_handlers`` — these tests pin the +seam (deps wiring, exception swallow contract, return shapes, +guard short-circuits) so future drift fails here, not at runtime +against real executors / clients.""" + +from __future__ import annotations + +import os +import threading +from typing import Any, Dict, List + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.database_update import ( + auto_start_database_update, auto_deep_scan_library, +) +from core.automation.handlers.duplicate_cleaner import auto_run_duplicate_cleaner +from core.automation.handlers.quality_scanner import auto_start_quality_scan +from core.automation.handlers.maintenance import ( + auto_clear_quarantine, auto_cleanup_wishlist, + auto_update_discovery_pool, auto_backup_database, +) +from core.automation.handlers.download_cleanup import ( + auto_clean_search_history, auto_clean_completed_downloads, +) +from core.automation.handlers.run_script import auto_run_script +from core.automation.handlers.search_and_download import auto_search_and_download + + +class _StubLogger: + def debug(self, *a, **k): pass + def info(self, *a, **k): pass + def warning(self, *a, **k): pass + def error(self, *a, **k): pass + + +class _StubConfig: + def __init__(self, values=None): + self._values = values or {} + + def get(self, key, default=None): + return self._values.get(key, default) + + def get_active_media_server(self): + return 'plex' + + +def _build_deps(**overrides) -> AutomationDeps: + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=_StubConfig(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── database_update / deep_scan ────────────────────────────────────── + + +class _StubExecutor: + def __init__(self): + self.submits: List[tuple] = [] + + def submit(self, fn, *args, **kwargs): + self.submits.append((fn, args, kwargs)) + + +class TestDatabaseUpdate: + def test_already_running_returns_skipped(self): + state = {'status': 'running'} + deps = _build_deps(get_db_update_state=lambda: state) + result = auto_start_database_update({}, deps) + assert result == {'status': 'skipped', 'reason': 'Database update already running'} + + def test_set_db_update_automation_id_called(self): + # Handler must propagate the automation id through the deps + # setter so the legacy global stays in sync. + captured: List[Any] = [] + state = {'status': 'idle'} + # Make the polling loop terminate immediately by flipping + # the status as soon as it's set to 'running'. + executor = _StubExecutor() + + def fake_task(*_a, **_k): + state['status'] = 'finished' + + executor.submit = lambda fn, *a, **k: fake_task() + deps = _build_deps( + get_db_update_state=lambda: state, + db_update_executor=executor, + set_db_update_automation_id=lambda v: captured.append(v), + run_db_update_task=fake_task, + ) + # Replace time.sleep so we don't actually wait + import core.automation.handlers.database_update as module + original = module.time.sleep + module.time.sleep = lambda _: None + try: + result = auto_start_database_update({'_automation_id': 'auto-1'}, deps) + finally: + module.time.sleep = original + assert captured == ['auto-1'] + assert result['status'] == 'completed' + + +class TestDeepScan: + def test_already_running_returns_skipped(self): + state = {'status': 'running'} + deps = _build_deps(get_db_update_state=lambda: state) + result = auto_deep_scan_library({}, deps) + assert result == {'status': 'skipped', 'reason': 'Database update already running'} + + +# ─── duplicate_cleaner ──────────────────────────────────────────────── + + +class TestDuplicateCleaner: + def test_already_running_returns_skipped(self): + state = {'status': 'running'} + deps = _build_deps(get_duplicate_cleaner_state=lambda: state) + result = auto_run_duplicate_cleaner({}, deps) + assert result == {'status': 'skipped', 'reason': 'Duplicate cleaner already running'} + + +# ─── quality_scanner ────────────────────────────────────────────────── + + +class TestQualityScanner: + def test_already_running_returns_skipped(self): + state = {'status': 'running'} + deps = _build_deps(get_quality_scanner_state=lambda: state) + result = auto_start_quality_scan({}, deps) + assert result == {'status': 'skipped', 'reason': 'Quality scan already running'} + + +# ─── clear_quarantine ──────────────────────────────────────────────── + + +class TestClearQuarantine: + def test_no_quarantine_folder_returns_zero(self, tmp_path): + # Point at a non-existent path. + deps = _build_deps( + config_manager=_StubConfig({'soulseek.download_path': str(tmp_path / 'nonexistent')}), + docker_resolve_path=lambda p: p, + ) + result = auto_clear_quarantine({}, deps) + assert result == {'status': 'completed', 'removed': '0'} + + def test_clears_files_from_quarantine(self, tmp_path): + download_path = tmp_path + quarantine_path = download_path / 'ss_quarantine' + quarantine_path.mkdir() + (quarantine_path / 'a.flac').write_bytes(b'') + (quarantine_path / 'b.flac').write_bytes(b'') + deps = _build_deps( + config_manager=_StubConfig({'soulseek.download_path': str(download_path)}), + docker_resolve_path=lambda p: p, + ) + result = auto_clear_quarantine({}, deps) + assert result == {'status': 'completed', 'removed': '2'} + + +# ─── cleanup_wishlist ──────────────────────────────────────────────── + + +class TestCleanupWishlist: + def test_returns_count_from_db(self): + class _DB: + def remove_wishlist_duplicates(self, profile_id): + assert profile_id == 1 + return 7 + + deps = _build_deps(get_database=lambda: _DB()) + result = auto_cleanup_wishlist({}, deps) + assert result == {'status': 'completed', 'removed': '7'} + + def test_returns_zero_when_db_returns_falsey(self): + class _DB: + def remove_wishlist_duplicates(self, profile_id): + return None + + deps = _build_deps(get_database=lambda: _DB()) + result = auto_cleanup_wishlist({}, deps) + assert result == {'status': 'completed', 'removed': '0'} + + +# ─── update_discovery_pool ─────────────────────────────────────────── + + +class TestUpdateDiscoveryPool: + def test_success(self): + called: List[Any] = [] + + class _Scanner: + def update_discovery_pool_incremental(self, profile_id): + called.append(profile_id) + + deps = _build_deps( + get_watchlist_scanner=lambda spc: _Scanner(), + get_current_profile_id=lambda: 7, + ) + result = auto_update_discovery_pool({}, deps) + assert result['status'] == 'completed' + assert result['_manages_own_progress'] is True + assert called == [7] + + def test_exception_swallowed(self): + def boom(_): + raise RuntimeError('scanner down') + + deps = _build_deps(get_watchlist_scanner=boom) + result = auto_update_discovery_pool({}, deps) + assert result['status'] == 'error' + assert 'scanner down' in result['reason'] + + +# ─── backup_database ───────────────────────────────────────────────── + + +class TestBackupDatabase: + def test_missing_database_returns_error(self, tmp_path, monkeypatch): + monkeypatch.setenv('DATABASE_PATH', str(tmp_path / 'no.db')) + deps = _build_deps() + result = auto_backup_database({}, deps) + assert result == {'status': 'error', 'reason': 'Database file not found'} + + +# ─── clean_search_history ──────────────────────────────────────────── + + +class TestCleanSearchHistory: + def test_soulseek_inactive_skips(self): + deps = _build_deps( + config_manager=_StubConfig({'download_source.mode': 'tidal'}), + ) + result = auto_clean_search_history({}, deps) + assert result == {'status': 'skipped'} + + def test_no_orchestrator_skips(self): + deps = _build_deps( + config_manager=_StubConfig({'download_source.mode': 'soulseek'}), + download_orchestrator=None, + ) + result = auto_clean_search_history({}, deps) + assert result == {'status': 'skipped'} + + +# ─── clean_completed_downloads ─────────────────────────────────────── + + +class TestCleanCompletedDownloads: + def test_active_batches_skip(self): + # Active batch present → handler returns 'completed' without doing anything. + deps = _build_deps( + get_download_batches=lambda: {'b1': {'phase': 'downloading'}}, + get_download_tasks=lambda: {}, + ) + result = auto_clean_completed_downloads({}, deps) + assert result == {'status': 'completed'} + + +# ─── run_script ────────────────────────────────────────────────────── + + +class TestRunScript: + def test_no_script_name_returns_error(self): + deps = _build_deps() + result = auto_run_script({}, deps) + assert result == {'status': 'error', 'error': 'No script selected'} + + def test_path_traversal_blocked(self, tmp_path): + scripts_dir = tmp_path / 'scripts' + scripts_dir.mkdir() + # Place a script OUTSIDE the scripts dir + try to reach it + # via ../ traversal. + evil = tmp_path / 'evil.sh' + evil.write_text('#!/bin/bash\necho evil') + deps = _build_deps( + config_manager=_StubConfig({'scripts.path': str(scripts_dir)}), + docker_resolve_path=lambda p: p, + ) + result = auto_run_script({'script_name': '../evil.sh'}, deps) + assert result['status'] == 'error' + assert 'path traversal' in result['error'].lower() + + def test_missing_script_returns_error(self, tmp_path): + scripts_dir = tmp_path / 'scripts' + scripts_dir.mkdir() + deps = _build_deps( + config_manager=_StubConfig({'scripts.path': str(scripts_dir)}), + ) + result = auto_run_script({'script_name': 'no_such_script.sh'}, deps) + assert result['status'] == 'error' + assert 'not found' in result['error'].lower() + + +# ─── search_and_download ───────────────────────────────────────────── + + +class TestSearchAndDownload: + def test_no_query_returns_error(self): + deps = _build_deps() + result = auto_search_and_download({}, deps) + assert result == {'status': 'error', 'error': 'No search query provided'} + + def test_query_from_event_data_used(self): + captured_queries: List[str] = [] + + class _Orchestrator: + async def search_and_download_best(self, q): + captured_queries.append(q) + return 'dl-id-123' + + deps = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda coro: 'dl-id-123', + ) + result = auto_search_and_download( + {'_event_data': {'query': 'Adele Hello'}}, deps, + ) + assert result['status'] == 'completed' + assert result['query'] == 'Adele Hello' + assert result['download_id'] == 'dl-id-123' + + def test_no_match_returns_not_found(self): + class _Orchestrator: + async def search_and_download_best(self, q): + return None + + deps = _build_deps( + download_orchestrator=_Orchestrator(), + run_async=lambda coro: None, + ) + result = auto_search_and_download({'query': 'xyz'}, deps) + assert result['status'] == 'not_found' + assert result['query'] == 'xyz' diff --git a/tests/automation/test_handlers_personalized_pipeline.py b/tests/automation/test_handlers_personalized_pipeline.py new file mode 100644 index 00000000..049cd466 --- /dev/null +++ b/tests/automation/test_handlers_personalized_pipeline.py @@ -0,0 +1,529 @@ +"""Boundary tests for the personalized playlist pipeline handler. + +Pin every shape: empty kinds error, refresh_first behaviour, snapshot +load + sync dispatch, missing-tracks skip, exception swallowing, +pipeline_running flag cleanup, sync payload shape passed to +_run_sync_task.""" + +from __future__ import annotations + +import threading +from types import SimpleNamespace +from typing import Any, List + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.personalized_pipeline import ( + auto_personalized_pipeline, + _build_payloads_for_kinds, + _track_to_sync_shape, + _sync_personalized_playlist, +) + + +class _StubLogger: + def debug(self, *a, **k): pass + def info(self, *a, **k): pass + def warning(self, *a, **k): pass + def error(self, *a, **k): pass + + +def _build_deps(**overrides) -> AutomationDeps: + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=object(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── Track shape converter ─────────────────────────────────────────── + + +class TestTrackToSyncShape: + def test_basic_shape(self): + track = SimpleNamespace( + track_name='Song', artist_name='Artist', album_name='Album', + spotify_track_id='sp-1', itunes_track_id=None, deezer_track_id=None, + duration_ms=200000, + ) + out = _track_to_sync_shape(track) + assert out == { + 'name': 'Song', + 'artists': [{'name': 'Artist'}], + 'album': {'name': 'Album'}, + 'duration_ms': 200000, + 'id': 'sp-1', + } + + def test_falls_back_through_source_ids(self): + t1 = SimpleNamespace(track_name='', artist_name='', album_name='', + spotify_track_id=None, itunes_track_id='it-1', + deezer_track_id=None, duration_ms=0) + assert _track_to_sync_shape(t1)['id'] == 'it-1' + + t2 = SimpleNamespace(track_name='', artist_name='', album_name='', + spotify_track_id=None, itunes_track_id=None, + deezer_track_id='dz-1', duration_ms=0) + assert _track_to_sync_shape(t2)['id'] == 'dz-1' + + def test_no_id_returns_empty_string(self): + t = SimpleNamespace(track_name='X', artist_name='Y', album_name='Z', + spotify_track_id=None, itunes_track_id=None, + deezer_track_id=None, duration_ms=0) + assert _track_to_sync_shape(t)['id'] == '' + + def test_preserves_enriched_track_data_for_wishlist_metadata(self): + track = SimpleNamespace( + track_name='Bare Name', artist_name='Bare Artist', album_name='Bare Album', + spotify_track_id='sp-rich', itunes_track_id=None, deezer_track_id=None, + album_cover_url=None, duration_ms=200000, popularity=33, + track_data_json={ + 'id': 'sp-rich', + 'name': 'Rich Name', + 'artists': [{'name': 'Rich Artist', 'id': 'artist-1'}], + 'album': { + 'id': 'album-1', + 'name': 'Rich Album', + 'images': [{'url': 'https://example.test/cover.jpg'}], + }, + 'duration_ms': 201000, + 'preview_url': 'https://example.test/preview.mp3', + }, + ) + + out = _track_to_sync_shape(track) + + assert out['name'] == 'Rich Name' + assert out['artists'][0] == {'name': 'Rich Artist', 'id': 'artist-1'} + assert out['album']['id'] == 'album-1' + assert out['album']['images'][0]['url'] == 'https://example.test/cover.jpg' + assert out['preview_url'] == 'https://example.test/preview.mp3' + + def test_album_cover_url_fills_album_images_when_no_rich_blob(self): + track = SimpleNamespace( + track_name='Song', artist_name='Artist', album_name='Album', + spotify_track_id='sp-1', itunes_track_id=None, deezer_track_id=None, + album_cover_url='https://example.test/fallback.jpg', + duration_ms=200000, track_data_json=None, + ) + + out = _track_to_sync_shape(track) + + assert out['album'] == { + 'name': 'Album', + 'images': [{'url': 'https://example.test/fallback.jpg'}], + } + + +# ─── Empty / config validation ────────────────────────────────────── + + +class TestEmptyConfig: + def test_no_kinds_returns_error_and_clears_flag(self): + deps = _build_deps() + deps.state.set_pipeline_running(True) # simulate already-running + result = auto_personalized_pipeline({}, deps) + assert result['status'] == 'error' + assert 'No personalized playlist' in result['error'] + assert deps.state.pipeline_running is False + + def test_empty_kinds_list_returns_error(self): + deps = _build_deps() + result = auto_personalized_pipeline({'kinds': []}, deps) + assert result['status'] == 'error' + assert deps.state.pipeline_running is False + + def test_non_list_kinds_returns_error(self): + deps = _build_deps() + result = auto_personalized_pipeline({'kinds': 'not_a_list'}, deps) + assert result['status'] == 'error' + + +# ─── Payload building ─────────────────────────────────────────────── + + +class _StubManagerNoTracks: + def ensure_playlist(self, kind, variant, profile_id): + # last_generated_at non-None so pipeline treats the snapshot as + # already-generated-but-empty (rather than first-run-needs-gen). + return SimpleNamespace( + id=1, name=f'{kind}-{variant}', kind=kind, variant=variant, + is_stale=False, last_generated_at='2026-05-15T20:00:00', + ) + + def refresh_playlist(self, kind, variant, profile_id): + return self.ensure_playlist(kind, variant, profile_id) + + def get_playlist_tracks(self, playlist_id): + return [] + + +class _StubManagerWithTracks: + def __init__(self, tracks_per_kind=None): + self.tracks_per_kind = tracks_per_kind or {} + self.refresh_calls: List[tuple] = [] + self.ensure_calls: List[tuple] = [] + + def ensure_playlist(self, kind, variant, profile_id): + self.ensure_calls.append((kind, variant, profile_id)) + return SimpleNamespace( + id=hash((kind, variant)) % 10000, + name=f'{kind}-{variant or "S"}', kind=kind, variant=variant, + is_stale=False, last_generated_at='2026-05-15T20:00:00', + ) + + def refresh_playlist(self, kind, variant, profile_id): + self.refresh_calls.append((kind, variant, profile_id)) + # Mirror real manager: refresh returns a record without invoking + # the public ensure_playlist API path again. + return SimpleNamespace( + id=hash((kind, variant)) % 10000, + name=f'{kind}-{variant or "S"}', kind=kind, variant=variant, + is_stale=False, last_generated_at='2026-05-15T20:00:00', + ) + + def get_playlist_tracks(self, playlist_id): + # Return all tracks regardless of id — tests scope to one playlist at a time. + for tracks in self.tracks_per_kind.values(): + if tracks: + return [SimpleNamespace( + track_name=t['name'], artist_name=t.get('artist', 'A'), + album_name=t.get('album', 'Al'), + spotify_track_id=t.get('id'), + itunes_track_id=None, deezer_track_id=None, + duration_ms=200000, + ) for t in tracks] + return [] + + +class TestPayloadBuilding: + def test_skips_kinds_with_no_tracks(self): + deps = _build_deps() + manager = _StubManagerNoTracks() + payloads = _build_payloads_for_kinds( + deps, manager, + [{'kind': 'hidden_gems'}, {'kind': 'discovery_shuffle'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert payloads == [] + + def test_skips_invalid_entries(self): + deps = _build_deps() + manager = _StubManagerNoTracks() + payloads = _build_payloads_for_kinds( + deps, manager, + ['not-a-dict', {}, {'variant': 'no-kind'}], # all invalid + profile_id=1, automation_id=None, refresh_first=False, + ) + assert payloads == [] + + def test_refresh_first_calls_refresh(self): + deps = _build_deps() + manager = _StubManagerWithTracks( + tracks_per_kind={'hidden_gems': [{'name': 'T', 'id': 'sp-1'}]}, + ) + _build_payloads_for_kinds( + deps, manager, + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=True, + ) + assert manager.refresh_calls == [('hidden_gems', '', 1)] + assert manager.ensure_calls == [] + + def test_no_refresh_calls_ensure(self): + deps = _build_deps() + manager = _StubManagerWithTracks( + tracks_per_kind={'hidden_gems': [{'name': 'T', 'id': 'sp-1'}]}, + ) + _build_payloads_for_kinds( + deps, manager, + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert manager.ensure_calls == [('hidden_gems', '', 1)] + assert manager.refresh_calls == [] + + def test_payload_shape(self): + deps = _build_deps() + manager = _StubManagerWithTracks( + tracks_per_kind={'hidden_gems': [ + {'name': 'Track1', 'id': 'sp-1'}, + {'name': 'Track2', 'id': 'sp-2'}, + ]}, + ) + payloads = _build_payloads_for_kinds( + deps, manager, + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert len(payloads) == 1 + p = payloads[0] + assert p['kind'] == 'hidden_gems' + assert p['variant'] == '' + assert p['name'] == 'hidden_gems-S' + assert p['sync_id'].startswith('auto_personalized_hidden_gems_') + assert len(p['tracks_json']) == 2 + assert p['tracks_json'][0]['id'] == 'sp-1' + + def test_stale_snapshot_auto_refreshes_even_without_refresh_first(self): + """When the manager reports is_stale=True, the pipeline refreshes + regardless of the refresh_first config flag — the source data + (discovery_pool / curated lists) changed, so the snapshot must + be regenerated before syncing or we'd push stale data.""" + deps = _build_deps() + # Stub manager whose ensure_playlist returns a stale record. + # refresh_playlist should still get called. + refresh_called = [] + + class _StaleMgr: + def ensure_playlist(self, kind, variant, profile_id): + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, is_stale=True, + last_generated_at='2026-05-15T20:00:00', + ) + def refresh_playlist(self, kind, variant, profile_id): + refresh_called.append((kind, variant)) + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, is_stale=False, + last_generated_at='2026-05-15T20:00:00', + ) + def get_playlist_tracks(self, _id): + return [SimpleNamespace( + track_name='Refreshed', artist_name='A', album_name='Al', + spotify_track_id='sp-fresh', itunes_track_id=None, + deezer_track_id=None, duration_ms=200000, + )] + + payloads = _build_payloads_for_kinds( + deps, _StaleMgr(), + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert refresh_called == [('hidden_gems', '')] + assert len(payloads) == 1 + assert payloads[0]['tracks_json'][0]['name'] == 'Refreshed' + + def test_non_stale_snapshot_skips_refresh(self): + """When the snapshot is fresh AND refresh_first is False, just + read the existing tracks without re-running the generator.""" + deps = _build_deps() + refresh_called = [] + + class _FreshMgr: + def ensure_playlist(self, kind, variant, profile_id): + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, is_stale=False, + last_generated_at='2026-05-15T20:00:00', + ) + def refresh_playlist(self, *_a, **_k): + refresh_called.append('called') + return SimpleNamespace( + id=1, name='x', kind='x', variant='', is_stale=False, + last_generated_at='2026-05-15T20:00:00', + ) + def get_playlist_tracks(self, _id): + return [SimpleNamespace( + track_name='Cached', artist_name='A', album_name='Al', + spotify_track_id='sp-1', itunes_track_id=None, + deezer_track_id=None, duration_ms=200000, + )] + + _build_payloads_for_kinds( + deps, _FreshMgr(), + [{'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert refresh_called == [] + + def test_never_generated_snapshot_triggers_first_refresh(self): + """First-run case: pipeline picks a brand-new kind, ensure_playlist + auto-creates the row with track_count=0 and last_generated_at=None. + Without this branch the pipeline would read the empty snapshot and + silently skip — user picked a kind and got nothing. With the branch, + last_generated_at=None forces a refresh so the generator actually runs.""" + deps = _build_deps() + refresh_called = [] + + class _NeverGenMgr: + def ensure_playlist(self, kind, variant, profile_id): + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, + is_stale=False, last_generated_at=None, + ) + def refresh_playlist(self, kind, variant, profile_id): + refresh_called.append((kind, variant)) + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, + is_stale=False, last_generated_at='2026-05-15T20:00:00', + ) + def get_playlist_tracks(self, _id): + return [SimpleNamespace( + track_name='Generated', artist_name='A', album_name='Al', + spotify_track_id='sp-new', itunes_track_id=None, + deezer_track_id=None, duration_ms=200000, + )] + + payloads = _build_payloads_for_kinds( + deps, _NeverGenMgr(), + [{'kind': 'fresh_tape'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert refresh_called == [('fresh_tape', '')] + assert len(payloads) == 1 + assert payloads[0]['tracks_json'][0]['name'] == 'Generated' + + def test_manager_exception_swallowed_continues_to_next(self): + deps = _build_deps() + + class _ExplodingMgr: + def __init__(self): + self.calls = [] + def ensure_playlist(self, kind, variant, profile_id): + self.calls.append(kind) + if kind == 'broken': + raise RuntimeError('manager boom') + return SimpleNamespace( + id=1, name=kind, kind=kind, variant=variant, is_stale=False, + last_generated_at='2026-05-15T20:00:00', + ) + def get_playlist_tracks(self, _id): + return [] + + mgr = _ExplodingMgr() + # broken raises, hidden_gems proceeds (just no tracks). + payloads = _build_payloads_for_kinds( + deps, mgr, + [{'kind': 'broken'}, {'kind': 'hidden_gems'}], + profile_id=1, automation_id=None, refresh_first=False, + ) + assert mgr.calls == ['broken', 'hidden_gems'] + assert payloads == [] # neither produced tracks + + +# ─── Sync launch ──────────────────────────────────────────────────── + + +class TestSyncLaunch: + def test_sync_one_playlist_starts_thread(self): + captured: List[tuple] = [] + + def fake_run_sync_task(*args): + captured.append(args) + + deps = _build_deps( + run_sync_task=fake_run_sync_task, + get_current_profile_id=lambda: 7, + ) + payload = { + 'sync_id': 'auto_personalized_hidden_gems_', + 'name': 'Hidden Gems', + 'tracks_json': [{'name': 'X', 'id': 'sp-1'}], + 'image_url': '', + } + result = _sync_personalized_playlist(deps, payload) + assert result['status'] == 'started' + # Wait for thread to invoke fake_run_sync_task. + for _ in range(100): + if captured: + break + import time + time.sleep(0.01) + assert len(captured) == 1 + # Args: (sync_id, name, tracks_json, automation_id, profile_id, image_url) + assert captured[0][0] == 'auto_personalized_hidden_gems_' + assert captured[0][1] == 'Hidden Gems' + assert captured[0][3] is None # automation_id muted + assert captured[0][4] == 7 # profile_id + + +# ─── Full pipeline (with stubbed manager + sync states) ───────────── + + +class TestPipelineHappyPath: + def test_pipeline_completes_with_synced_count(self): + # Stub manager returns one playlist with 2 tracks. + manager = _StubManagerWithTracks( + tracks_per_kind={'hidden_gems': [ + {'name': 'A', 'id': 'sp-1'}, + {'name': 'B', 'id': 'sp-2'}, + ]}, + ) + + # sync_states populated as if the sync background task finished. + sync_states_storage = {} + + def fake_run_sync(sync_id, name, tracks, aid, pid, img): + sync_states_storage[sync_id] = { + 'status': 'finished', + 'result': {'matched_tracks': 2}, + } + + deps = _build_deps( + build_personalized_manager=lambda: manager, + run_sync_task=fake_run_sync, + get_sync_states=lambda: sync_states_storage, + ) + # Patch time.sleep in shared helper so test doesn't take 2s per iter. + import core.automation.handlers._pipeline_shared as shared + orig = shared.time.sleep + shared.time.sleep = lambda _: None + try: + result = auto_personalized_pipeline( + {'_automation_id': 'auto-1', 'kinds': [{'kind': 'hidden_gems'}]}, + deps, + ) + finally: + shared.time.sleep = orig + assert result['status'] == 'completed' + assert result['_manages_own_progress'] is True + # Pipeline-running flag cleaned up. + assert deps.state.pipeline_running is False diff --git a/tests/automation/test_handlers_playlist.py b/tests/automation/test_handlers_playlist.py new file mode 100644 index 00000000..1a1548ef --- /dev/null +++ b/tests/automation/test_handlers_playlist.py @@ -0,0 +1,400 @@ +"""Boundary tests for the playlist-lifecycle automation handlers +(``refresh_mirrored``, ``sync_playlist``, ``discover_playlist``, +``playlist_pipeline``). + +The handlers themselves are mechanical lifts of the closures that +used to live in ``web_server._register_automation_handlers`` — these +tests pin the seam so the wiring stays correct (deps are read from +the deps object, not module-level globals; cross-handler calls in +the pipeline still compose; failure paths still return clear status +shapes). + +Source-specific branches inside ``refresh_mirrored`` (Spotify auth ++ public-embed fallback, Deezer / Tidal / YouTube) are validated +end-to-end via fake clients in ``deps`` rather than per-source +because they're a verbatim lift — drift would show up here as a +behavior change.""" + +from __future__ import annotations + +import json +import threading +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.discover_playlist import auto_discover_playlist +from core.automation.handlers.playlist_pipeline import auto_playlist_pipeline +from core.automation.handlers.refresh_mirrored import auto_refresh_mirrored +from core.automation.handlers.sync_playlist import auto_sync_playlist + + +# ─── shared scaffolding ────────────────────────────────────────────── + + +class _StubLogger: + def __init__(self): + self.messages: List[tuple] = [] + + def debug(self, *a, **k): self.messages.append(('debug', a)) + def info(self, *a, **k): self.messages.append(('info', a)) + def warning(self, *a, **k): self.messages.append(('warning', a)) + def error(self, *a, **k): self.messages.append(('error', a)) + + +@dataclass +class _StubDB: + """Fake MusicDatabase — minimal surface used by the playlist handlers.""" + + playlists: List[dict] = field(default_factory=list) + playlist_tracks: Dict[int, List[dict]] = field(default_factory=dict) + extra_data_maps: Dict[int, Dict[str, str]] = field(default_factory=dict) + mirror_calls: List[dict] = field(default_factory=list) + + def get_mirrored_playlists(self) -> list: + return list(self.playlists) + + def get_mirrored_playlist(self, playlist_id: int) -> Optional[dict]: + for p in self.playlists: + if int(p.get('id', -1)) == int(playlist_id): + return p + return None + + def get_mirrored_playlist_tracks(self, playlist_id: int) -> list: + return list(self.playlist_tracks.get(int(playlist_id), [])) + + def get_mirrored_tracks_extra_data_map(self, playlist_id: int) -> dict: + return dict(self.extra_data_maps.get(int(playlist_id), {})) + + def mirror_playlist(self, **kwargs) -> None: + self.mirror_calls.append(kwargs) + + +def _build_deps(**overrides) -> AutomationDeps: + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=object(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: _StubDB(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── discover_playlist ─────────────────────────────────────────────── + + +class TestDiscoverPlaylist: + def test_no_playlist_id_returns_error(self): + deps = _build_deps() + result = auto_discover_playlist({}, deps) + assert result == {'status': 'error', 'reason': 'No playlist specified'} + + def test_specific_playlist_id_starts_worker(self): + db = _StubDB(playlists=[{'id': 42, 'name': 'Test Playlist'}]) + called: List[Any] = [] + deps = _build_deps( + get_database=lambda: db, + run_playlist_discovery_worker=lambda *a, **k: called.append((a, k)), + ) + result = auto_discover_playlist({'playlist_id': '42', '_automation_id': 'auto-1'}, deps) + assert result['status'] == 'started' + assert result['_manages_own_progress'] is True + assert result['playlist_count'] == '1' + # Worker spawned on a thread; give it a moment. + for _ in range(50): + if called: + break + import time + time.sleep(0.01) + assert len(called) == 1 + + def test_all_playlists_includes_every_one(self): + db = _StubDB(playlists=[ + {'id': 1, 'name': 'A'}, {'id': 2, 'name': 'B'}, {'id': 3, 'name': 'C'}, + ]) + deps = _build_deps(get_database=lambda: db) + result = auto_discover_playlist({'all': True}, deps) + assert result['playlist_count'] == '3' + assert 'A' in result['playlists'] + assert 'B' in result['playlists'] + assert 'C' in result['playlists'] + + def test_no_playlists_in_db_returns_error(self): + deps = _build_deps(get_database=lambda: _StubDB(playlists=[])) + result = auto_discover_playlist({'all': True}, deps) + assert result == {'status': 'error', 'reason': 'No playlists found'} + + +# ─── refresh_mirrored ──────────────────────────────────────────────── + + +@dataclass +class _StubSpotifyTrack: + id: str + name: str + artists: list + album: str + duration_ms: int + image_url: Optional[str] = None + + +@dataclass +class _StubSpotifyPlaylist: + tracks: list + + +class _StubSpotifyClient: + def __init__(self, playlist): + self._playlist = playlist + self._authenticated = True + + def is_spotify_authenticated(self) -> bool: + return self._authenticated + + def get_playlist_by_id(self, _source_id): + return self._playlist + + +class TestRefreshMirrored: + def test_no_playlist_specified_returns_error(self): + deps = _build_deps() + result = auto_refresh_mirrored({}, deps) + assert result == {'status': 'error', 'reason': 'No playlist specified'} + + def test_filters_unrefreshable_sources(self): + # Sources 'file' and 'beatport' have no API to refresh from. + db = _StubDB(playlists=[ + {'id': 1, 'name': 'F', 'source': 'file', 'source_playlist_id': '1'}, + {'id': 2, 'name': 'B', 'source': 'beatport', 'source_playlist_id': '2'}, + ]) + deps = _build_deps(get_database=lambda: db) + result = auto_refresh_mirrored({'all': True}, deps) + assert result['status'] == 'completed' + assert result['refreshed'] == '0' + assert db.mirror_calls == [] # nothing got pushed to DB + + def test_spotify_refresh_writes_to_db(self): + track = _StubSpotifyTrack( + id='track123', name='Hello', artists=['Adele'], + album='25', duration_ms=295000, + ) + playlist = _StubSpotifyPlaylist(tracks=[track]) + spotify = _StubSpotifyClient(playlist) + db = _StubDB(playlists=[ + {'id': 5, 'name': 'My Spot', 'source': 'spotify', + 'source_playlist_id': 'spot-id', 'profile_id': 1}, + ]) + deps = _build_deps(get_database=lambda: db, spotify_client=spotify) + result = auto_refresh_mirrored({'playlist_id': '5'}, deps) + assert result['status'] == 'completed' + assert result['refreshed'] == '1' + assert len(db.mirror_calls) == 1 + call = db.mirror_calls[0] + assert call['source'] == 'spotify' + assert call['source_playlist_id'] == 'spot-id' + assert call['name'] == 'My Spot' + assert len(call['tracks']) == 1 + # Spotify-source tracks should be auto-marked discovered. + extra = json.loads(call['tracks'][0]['extra_data']) + assert extra['discovered'] is True + assert extra['provider'] == 'spotify' + assert extra['matched_data']['id'] == 'track123' + + def test_per_playlist_exception_collected_into_errors(self): + # Force an exception by making the DB blow up on mirror_playlist. + class _ExplodingDB(_StubDB): + def mirror_playlist(self, **kwargs): + raise RuntimeError('db disk full') + + track = _StubSpotifyTrack(id='t', name='t', artists=['a'], album='a', duration_ms=0) + spotify = _StubSpotifyClient(_StubSpotifyPlaylist(tracks=[track])) + db = _ExplodingDB(playlists=[ + {'id': 1, 'name': 'X', 'source': 'spotify', 'source_playlist_id': 'spot'}, + ]) + deps = _build_deps(get_database=lambda: db, spotify_client=spotify) + result = auto_refresh_mirrored({'all': True}, deps) + # Error captured, status still 'completed' (handler returns counts). + assert result['status'] == 'completed' + assert result['errors'] == '1' + assert result['refreshed'] == '0' + + +# ─── sync_playlist ─────────────────────────────────────────────────── + + +class TestSyncPlaylist: + def test_no_playlist_id_returns_error(self): + deps = _build_deps() + result = auto_sync_playlist({}, deps) + assert result == {'status': 'error', 'reason': 'No playlist specified'} + + def test_playlist_not_found_returns_error(self): + deps = _build_deps(get_database=lambda: _StubDB(playlists=[])) + result = auto_sync_playlist({'playlist_id': '99'}, deps) + assert result == {'status': 'error', 'reason': 'Playlist not found'} + + def test_no_tracks_returns_error(self): + db = _StubDB(playlists=[{'id': 1, 'name': 'P'}], playlist_tracks={1: []}) + deps = _build_deps(get_database=lambda: db) + result = auto_sync_playlist({'playlist_id': '1'}, deps) + assert result == {'status': 'error', 'reason': 'No tracks in playlist'} + + def test_no_discovered_tracks_skips(self): + # All tracks lack discovery + spotify_hint + valid IDs. + db = _StubDB( + playlists=[{'id': 1, 'name': 'P'}], + playlist_tracks={1: [{}, {}]}, # empty tracks → nothing usable + ) + deps = _build_deps(get_database=lambda: db) + result = auto_sync_playlist({'playlist_id': '1'}, deps) + assert result['status'] == 'skipped' + assert 'No discovered tracks' in result['reason'] + assert result['skipped_tracks'] == '2' + + def test_discovered_track_starts_sync_thread(self): + discovered_track = { + 'extra_data': json.dumps({ + 'discovered': True, + 'matched_data': { + 'id': 'spot-1', 'name': 'Track', 'artists': [{'name': 'X'}], + 'album': {'name': 'Album'}, 'duration_ms': 200000, + }, + }), + 'artist_name': 'X', + } + db = _StubDB( + playlists=[{'id': 1, 'name': 'P'}], + playlist_tracks={1: [discovered_track]}, + ) + sync_calls: List[tuple] = [] + deps = _build_deps( + get_database=lambda: db, + run_sync_task=lambda *a, **k: sync_calls.append((a, k)), + ) + result = auto_sync_playlist({'playlist_id': '1'}, deps) + assert result['status'] == 'started' + assert result['_manages_own_progress'] is True + assert result['discovered_tracks'] == '1' + # Wait for thread to fire run_sync_task + for _ in range(50): + if sync_calls: + break + import time + time.sleep(0.01) + assert len(sync_calls) == 1 + + def test_unchanged_since_last_sync_returns_skipped(self): + discovered_track = { + 'extra_data': json.dumps({ + 'discovered': True, + 'matched_data': { + 'id': 'spot-1', 'name': 'T', 'artists': [{'name': 'X'}], + 'album': {'name': 'A'}, 'duration_ms': 0, + }, + }), + 'artist_name': 'X', + } + db = _StubDB( + playlists=[{'id': 1, 'name': 'P'}], + playlist_tracks={1: [discovered_track]}, + ) + + # Pre-populate the sync-status file with the EXPECTED hash so the + # preflight short-circuit fires. + import hashlib + expected_hash = hashlib.md5('spot-1'.encode()).hexdigest() + sync_statuses = { + 'auto_mirror_1': {'tracks_hash': expected_hash, 'matched_tracks': 1} + } + + deps = _build_deps( + get_database=lambda: db, + load_sync_status_file=lambda: sync_statuses, + ) + result = auto_sync_playlist({'playlist_id': '1'}, deps) + assert result['status'] == 'skipped' + assert 'unchanged' in result['reason'] + + +# ─── playlist_pipeline ─────────────────────────────────────────────── + + +class TestPlaylistPipeline: + def test_no_playlist_specified_returns_error(self): + deps = _build_deps() + result = auto_playlist_pipeline({}, deps) + assert result == {'status': 'error', 'error': 'No playlist specified'} + # Pipeline-running flag MUST be cleared on error so the guard + # doesn't block subsequent triggers. + assert deps.state.pipeline_running is False + + def test_no_refreshable_playlists_clears_running_flag(self): + db = _StubDB(playlists=[ + {'id': 1, 'name': 'F', 'source': 'file'}, + {'id': 2, 'name': 'B', 'source': 'beatport'}, + ]) + deps = _build_deps(get_database=lambda: db) + result = auto_playlist_pipeline({'all': True}, deps) + assert result == {'status': 'error', 'error': 'No refreshable playlists found'} + assert deps.state.pipeline_running is False + + def test_pipeline_clears_running_on_unhandled_exception(self): + # Force the database accessor to blow up after the early checks. + class _ExplodingDB(_StubDB): + def get_mirrored_playlists(self): + raise RuntimeError('db down') + + db = _ExplodingDB(playlists=[]) + deps = _build_deps(get_database=lambda: db) + result = auto_playlist_pipeline({'all': True}, deps) + assert result['status'] == 'error' + assert result['_manages_own_progress'] is True + assert deps.state.pipeline_running is False diff --git a/tests/automation/test_handlers_simple.py b/tests/automation/test_handlers_simple.py new file mode 100644 index 00000000..ac8b2ef4 --- /dev/null +++ b/tests/automation/test_handlers_simple.py @@ -0,0 +1,329 @@ +"""Boundary tests for the simple extracted automation handlers +(``process_wishlist``, ``scan_watchlist``, ``scan_library``). + +Each handler is tested as a pure function: real ``AutomationDeps`` +constructed with stub callables, no Flask, no DB, no media-server +clients. The tests exercise the success path, the guard paths +(handler short-circuits when another instance is running), the +exception-swallowing contract (handlers must NEVER raise into the +engine), and the mutable-state machinery for handlers that own a +flag in ``AutomationState``. + +Pre-extraction these closures lived inside +``web_server._register_automation_handlers`` and were essentially +un-testable — every test would have needed to spin up the whole +Flask app and stub a dozen module-level globals.""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from typing import Any, Callable, List, Optional + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.process_wishlist import auto_process_wishlist +from core.automation.handlers.scan_watchlist import auto_scan_watchlist +from core.automation.handlers.scan_library import auto_scan_library + + +# ─── shared test scaffolding ────────────────────────────────────────── + + +def _build_deps(**overrides: Any) -> AutomationDeps: + """Return a default `AutomationDeps` with no-op callables. Tests + pass ``overrides`` to install behaviour on the specific deps they + care about.""" + + class _StubLogger: + def debug(self, *_a, **_k): pass + def info(self, *_a, **_k): pass + def warning(self, *_a, **_k): pass + def error(self, *_a, **_k): pass + + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=object(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── process_wishlist ───────────────────────────────────────────────── + + +class TestProcessWishlist: + def test_success_returns_completed_status(self): + called: List[Any] = [] + + def stub(automation_id=None): + called.append(automation_id) + + deps = _build_deps(process_wishlist_automatically=stub) + result = auto_process_wishlist({'_automation_id': 'auto-1'}, deps) + assert result == {'status': 'completed'} + assert called == ['auto-1'] + + def test_passes_none_when_no_automation_id(self): + called: List[Any] = [] + + def stub(automation_id=None): + called.append(automation_id) + + deps = _build_deps(process_wishlist_automatically=stub) + result = auto_process_wishlist({}, deps) + assert result == {'status': 'completed'} + assert called == [None] + + def test_handler_swallows_exceptions(self): + def stub(**_kwargs): + raise RuntimeError('boom') + + deps = _build_deps(process_wishlist_automatically=stub) + result = auto_process_wishlist({'_automation_id': 'a'}, deps) + assert result == {'status': 'error', 'error': 'boom'} + + +# ─── scan_watchlist ────────────────────────────────────────────────── + + +class TestScanWatchlist: + def test_fresh_scan_reports_summary_stats(self): + # Worker reassigns the state dict mid-run — handler detects + # via id() change and reports stats. + states = [ + {'summary': {}}, + {'summary': { + 'total_artists': 5, + 'successful_scans': 4, + 'new_tracks_found': 12, + 'tracks_added_to_wishlist': 8, + }}, + ] + idx = {'i': 0} + + def get_state(): + return states[idx['i']] + + def stub(**_kwargs): + idx['i'] = 1 # simulate the worker swapping the dict + + deps = _build_deps( + process_watchlist_scan_automatically=stub, + get_watchlist_scan_state=get_state, + ) + result = auto_scan_watchlist({}, deps) + assert result == { + 'status': 'completed', + 'artists_scanned': 5, + 'successful_scans': 4, + 'new_tracks_found': 12, + 'tracks_added_to_wishlist': 8, + } + + def test_no_fresh_scan_returns_bare_completed(self): + # Same dict identity before and after = no fresh scan ran. + same_dict = {'summary': {'total_artists': 999}} + deps = _build_deps( + process_watchlist_scan_automatically=lambda **_k: None, + get_watchlist_scan_state=lambda: same_dict, + ) + result = auto_scan_watchlist({}, deps) + assert result == {'status': 'completed'} + + def test_handler_swallows_exceptions(self): + def stub(**_kwargs): + raise ValueError('no scanner') + + deps = _build_deps(process_watchlist_scan_automatically=stub) + result = auto_scan_watchlist({}, deps) + assert result == {'status': 'error', 'error': 'no scanner'} + + +# ─── scan_library ──────────────────────────────────────────────────── + + +@dataclass +class _StubScanManager: + """Minimal fake of ``web_scan_manager`` — records calls + lets + tests script its responses.""" + + request_responses: List[dict] = field(default_factory=list) + status_responses: List[dict] = field(default_factory=list) + request_calls: List[str] = field(default_factory=list) + + def request_scan(self, label: str) -> dict: + self.request_calls.append(label) + return self.request_responses.pop(0) if self.request_responses else {'status': 'queued'} + + def get_scan_status(self) -> dict: + return self.status_responses.pop(0) if self.status_responses else {'status': 'idle'} + + +class TestScanLibrary: + def test_no_scan_manager_returns_error(self): + deps = _build_deps(web_scan_manager=None) + result = auto_scan_library({'_automation_id': 'a'}, deps) + assert result == {'status': 'error', 'reason': 'Scan manager not available'} + + def test_already_tracked_returns_skipped(self): + # Pre-set the state flag — handler should short-circuit. + state = AutomationState() + state.scan_library_automation_id = 'someone-else' + scanner = _StubScanManager(request_responses=[{'status': 'queued'}]) + deps = _build_deps(state=state, web_scan_manager=scanner) + result = auto_scan_library({'_automation_id': 'a'}, deps) + assert result == {'status': 'skipped', 'reason': 'Scan already being tracked'} + assert scanner.request_calls == ['Automation trigger (additional batch)'] + + def test_scan_completes_normally(self): + # request_scan returns scheduled; first poll = scheduled; + # second poll = scanning; third poll = idle. + scanner = _StubScanManager( + request_responses=[{'status': 'scheduled', 'delay_seconds': 5}], + status_responses=[ + {'status': 'scheduled'}, + {'status': 'scanning', 'elapsed_seconds': 10, 'max_time_seconds': 100}, + {'status': 'idle'}, + ], + ) + progress: List[dict] = [] + + def stub_progress(automation_id, **kwargs): + progress.append({'aid': automation_id, **kwargs}) + + deps = _build_deps( + web_scan_manager=scanner, + update_progress=stub_progress, + ) + # Patch time.sleep so the test runs instantly. + import core.automation.handlers.scan_library as module + original = module.time.sleep + module.time.sleep = lambda _: None + try: + result = auto_scan_library({'_automation_id': 'auto-1'}, deps) + finally: + module.time.sleep = original + + assert result['status'] == 'completed' + assert result.get('_manages_own_progress') is True + # State flag cleaned up after run + assert deps.state.scan_library_automation_id is None + # Progress phases emitted: scheduled, scan-start, scanning, completed + statuses = [p.get('status') for p in progress] + assert 'finished' in statuses + + def test_state_cleanup_on_exception(self): + class ExplodingScanner: + def request_scan(self, _): + raise RuntimeError('boom') + + def get_scan_status(self): + return {'status': 'idle'} + + progress: List[dict] = [] + deps = _build_deps( + web_scan_manager=ExplodingScanner(), + update_progress=lambda aid, **kw: progress.append({'aid': aid, **kw}), + ) + result = auto_scan_library({'_automation_id': 'auto-x'}, deps) + assert result['status'] == 'error' + assert result['_manages_own_progress'] is True + # State flag still cleaned up + assert deps.state.scan_library_automation_id is None + # Error progress emitted + assert any(p.get('status') == 'error' for p in progress) + + +# ─── AutomationState ────────────────────────────────────────────────── + + +class TestAutomationState: + def test_default_state(self): + s = AutomationState() + assert s.scan_library_automation_id is None + assert s.db_update_automation_id is None + assert s.pipeline_running is False + assert s.is_scan_library_active() is False + assert s.is_pipeline_running() is False + + def test_set_scan_library_id(self): + s = AutomationState() + s.set_scan_library_id('auto-1') + assert s.scan_library_automation_id == 'auto-1' + assert s.is_scan_library_active() is True + s.set_scan_library_id(None) + assert s.is_scan_library_active() is False + + def test_set_pipeline_running(self): + s = AutomationState() + s.set_pipeline_running(True) + assert s.is_pipeline_running() is True + s.set_pipeline_running(False) + assert s.is_pipeline_running() is False + + def test_concurrent_set_safe_via_lock(self): + # Smoke test: two threads flipping the same field don't crash. + # Lock ensures the final value is consistent. + s = AutomationState() + + def worker(): + for _ in range(100): + s.set_pipeline_running(True) + s.set_pipeline_running(False) + + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert s.pipeline_running is False diff --git a/tests/automation/test_progress_callbacks.py b/tests/automation/test_progress_callbacks.py new file mode 100644 index 00000000..10456bfd --- /dev/null +++ b/tests/automation/test_progress_callbacks.py @@ -0,0 +1,244 @@ +"""Boundary tests for the progress + history callbacks extracted +from ``web_server._register_automation_handlers``. + +The callbacks are wired by the engine via ``register_progress_callbacks``; +each test invokes the extracted top-level function with stub deps +and verifies the right downstream call fires.""" + +from __future__ import annotations + +import threading +from typing import Any, Dict, List, Tuple + +import pytest + +from core.automation.deps import AutomationDeps, AutomationState +from core.automation.handlers.progress_callbacks import ( + progress_init, + progress_finish, + record_history, + on_library_scan_completed, + register_library_scan_completed_emitter, +) + + +def _build_deps(**overrides) -> AutomationDeps: + class _StubLogger: + def debug(self, *a, **k): pass + def info(self, *a, **k): pass + def warning(self, *a, **k): pass + def error(self, *a, **k): pass + + defaults = dict( + engine=object(), + state=AutomationState(), + config_manager=object(), + update_progress=lambda *a, **k: None, + logger=_StubLogger(), + get_database=lambda: object(), + spotify_client=None, + tidal_client=None, + web_scan_manager=None, + process_wishlist_automatically=lambda **k: None, + process_watchlist_scan_automatically=lambda **k: None, + is_wishlist_actually_processing=lambda: False, + is_watchlist_actually_scanning=lambda: False, + get_watchlist_scan_state=lambda: {}, + run_playlist_discovery_worker=lambda *a, **k: None, + run_sync_task=lambda *a, **k: None, + load_sync_status_file=lambda: {}, + get_deezer_client=lambda: None, + parse_youtube_playlist=lambda url: None, + get_sync_states=lambda: {}, + set_db_update_automation_id=lambda v: None, + get_db_update_state=lambda: {}, + db_update_lock=threading.Lock(), + db_update_executor=None, + run_db_update_task=lambda *a, **k: None, + run_deep_scan_task=lambda *a, **k: None, + get_duplicate_cleaner_state=lambda: {}, + duplicate_cleaner_lock=threading.Lock(), + duplicate_cleaner_executor=None, + run_duplicate_cleaner=lambda: None, + get_quality_scanner_state=lambda: {}, + quality_scanner_lock=threading.Lock(), + quality_scanner_executor=None, + run_quality_scanner=lambda *a, **k: None, + download_orchestrator=None, + run_async=lambda coro: None, + tasks_lock=threading.Lock(), + get_download_batches=lambda: {}, + get_download_tasks=lambda: {}, + sweep_empty_download_directories=lambda: 0, + get_staging_path=lambda: '/staging', + docker_resolve_path=lambda p: p, + get_current_profile_id=lambda: 1, + get_watchlist_scanner=lambda spc: None, + get_app=lambda: None, + get_beatport_data_cache=lambda: {'cache_lock': threading.Lock(), 'homepage': {}}, + init_automation_progress=lambda *a, **k: None, + record_progress_history=lambda *a, **k: None, + build_personalized_manager=lambda: None, + ) + defaults.update(overrides) + return AutomationDeps(**defaults) # type: ignore[arg-type] + + +# ─── progress_init ─────────────────────────────────────────────────── + + +class TestProgressInit: + def test_forwards_to_init_automation_progress(self): + captured: List[Tuple] = [] + + def fake(aid, name, action_type): + captured.append((aid, name, action_type)) + + deps = _build_deps(init_automation_progress=fake) + progress_init('auto-1', 'My Auto', 'wishlist', deps) + assert captured == [('auto-1', 'My Auto', 'wishlist')] + + +# ─── progress_finish ───────────────────────────────────────────────── + + +class TestProgressFinish: + def test_skips_when_handler_manages_own_progress(self): + # Handler set the flag — engine callback must NOT emit a + # second 'finished' over the top of the handler's own. + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda *a, **k: calls.append({'a': a, 'k': k})) + progress_finish('auto-1', {'_manages_own_progress': True, 'status': 'completed'}, deps) + assert calls == [] + + def test_completed_emits_finished_status(self): + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda aid, **kw: calls.append({'aid': aid, **kw})) + progress_finish('auto-1', {'status': 'completed'}, deps) + assert len(calls) == 1 + assert calls[0]['aid'] == 'auto-1' + assert calls[0]['status'] == 'finished' + assert calls[0]['progress'] == 100 + assert calls[0]['phase'] == 'Complete' + assert calls[0]['log_type'] == 'success' + + def test_error_status_emits_error_phase(self): + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda aid, **kw: calls.append({'aid': aid, **kw})) + progress_finish('auto-1', {'status': 'error', 'error': 'boom'}, deps) + assert calls[0]['status'] == 'error' + assert calls[0]['phase'] == 'Error' + assert calls[0]['log_line'] == 'boom' + assert calls[0]['log_type'] == 'error' + + def test_msg_falls_back_through_keys(self): + # error -> reason -> status -> 'done' + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda aid, **kw: calls.append({'aid': aid, **kw})) + progress_finish('auto-1', {'status': 'completed', 'reason': 'all good'}, deps) + assert calls[0]['log_line'] == 'all good' + + def test_msg_default_done(self): + calls: List[Dict] = [] + deps = _build_deps(update_progress=lambda aid, **kw: calls.append({'aid': aid, **kw})) + progress_finish('auto-1', {}, deps) + assert calls[0]['log_line'] == 'done' + + +# ─── record_history ────────────────────────────────────────────────── + + +class TestRecordHistory: + def test_passes_db_to_recorder(self): + captured: List[Tuple] = [] + db_obj = object() + deps = _build_deps( + get_database=lambda: db_obj, + record_progress_history=lambda aid, result, db: captured.append((aid, result, db)), + ) + record_history('auto-1', {'status': 'completed'}, deps) + assert captured == [('auto-1', {'status': 'completed'}, db_obj)] + + +# ─── on_library_scan_completed ─────────────────────────────────────── + + +class TestOnLibraryScanCompleted: + def test_no_engine_skips(self): + deps = _build_deps(engine=None) + # Should not raise. + on_library_scan_completed(deps) + + def test_emits_event_with_server_type(self): + emits: List[Tuple] = [] + + class _Engine: + def emit(self, name, payload): + emits.append((name, payload)) + + class _ScanMgr: + _current_server_type = 'plex' + + deps = _build_deps(engine=_Engine(), web_scan_manager=_ScanMgr()) + on_library_scan_completed(deps) + assert emits == [('library_scan_completed', {'server_type': 'plex'})] + + def test_unknown_server_type_when_attr_missing(self): + emits: List[Tuple] = [] + + class _Engine: + def emit(self, name, payload): + emits.append((name, payload)) + + deps = _build_deps(engine=_Engine(), web_scan_manager=object()) + on_library_scan_completed(deps) + assert emits[0][1] == {'server_type': 'unknown'} + + +# ─── register_library_scan_completed_emitter ───────────────────────── + + +class TestRegisterEmitter: + def test_no_scan_manager_noop(self): + # No web_scan_manager → no callback registered, no error. + deps = _build_deps(web_scan_manager=None) + register_library_scan_completed_emitter(deps) + + def test_registers_callback_with_scan_manager(self): + callbacks: List = [] + + class _ScanMgr: + _current_server_type = 'plex' + def add_scan_completion_callback(self, cb): + callbacks.append(cb) + + deps = _build_deps(web_scan_manager=_ScanMgr()) + register_library_scan_completed_emitter(deps) + assert len(callbacks) == 1 + # The registered callback must invoke without args (web_scan_manager + # calls completion callbacks with no params). + # Verify it does fire on_library_scan_completed when invoked. + emits: List = [] + + class _Engine: + def emit(self, name, payload): + emits.append((name, payload)) + + deps2 = _build_deps(engine=_Engine(), web_scan_manager=_ScanMgr()) + register_library_scan_completed_emitter(deps2) + # The lambda captured deps2; we need to grab the registered + # callback to invoke it. Re-register and capture. + captured = [] + class _Mgr2: + _current_server_type = 'jellyfin' + def add_scan_completion_callback(self, cb): + captured.append(cb) + deps3 = _build_deps(engine=_Engine(), web_scan_manager=_Mgr2()) + emits3 = [] + deps3 = _build_deps( + engine=type('E', (), {'emit': lambda self, n, p: emits3.append((n, p))})(), + web_scan_manager=_Mgr2(), + ) + register_library_scan_completed_emitter(deps3) + captured[0]() # invoke the registered callback + assert emits3 == [('library_scan_completed', {'server_type': 'jellyfin'})] diff --git a/tests/matching/test_version_mismatch.py b/tests/matching/test_version_mismatch.py new file mode 100644 index 00000000..3b81f788 --- /dev/null +++ b/tests/matching/test_version_mismatch.py @@ -0,0 +1,231 @@ +"""Boundary tests for ``core.matching.version_mismatch``. + +Pin every shape the version-mismatch escape valve has to handle so +future drift fails here instead of at runtime against a real download: +one-sided bare cases (the live-recording MB-metadata-gap that issue +#607 reported), two-sided real mismatches (live vs remix — keep +strict), high vs low fingerprint score gates, title/artist threshold +gates, defensive paths. +""" + +from __future__ import annotations + +import pytest + +from core.matching.version_mismatch import is_acceptable_version_mismatch + + +class TestEqualVersions: + def test_same_version_trivially_accepted(self): + # Equal version strings — no mismatch to decide. True. + assert is_acceptable_version_mismatch( + 'live', 'live', + fingerprint_score=0.0, + title_similarity=0.0, + artist_similarity=0.0, + ) is True + + def test_both_original_accepted(self): + assert is_acceptable_version_mismatch( + 'original', 'original', + fingerprint_score=0.0, + title_similarity=0.0, + artist_similarity=0.0, + ) is True + + +class TestOneSidedBareMismatch: + """Issue #607 example 2: expected has annotation, AcoustID's MB + record is bare. Accept when fingerprint + bare titles + artist + all line up.""" + + def test_live_vs_original_high_confidence_accepted(self): + # Reporter's exact case: "Clarity (Live at ...)" vs "Clarity" + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.95, + artist_similarity=1.0, + ) is True + + def test_original_vs_live_high_confidence_accepted(self): + # Same case in the other direction. + assert is_acceptable_version_mismatch( + 'original', 'live', + fingerprint_score=0.95, + title_similarity=0.95, + artist_similarity=1.0, + ) is True + + def test_live_at_thresholds_accepted(self): + # Exactly at the thresholds for the live-aware case. + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.85, + title_similarity=0.70, + artist_similarity=0.60, + ) is True + + +class TestNonLiveOneSidedMismatchStaysStrict: + """Other version markers (instrumental / remix / acoustic / etc) + have distinct fingerprints AND MB always annotates them in the + recording title. When AcoustID returns one of these for a bare + expected (or vice versa), the file genuinely IS that version — + the user asked for the wrong cut. Reject regardless of how high + the supporting scores are. + + This narrowness is what keeps the existing + test_acoustid_version_mismatch suite passing — instrumental + vs vocal etc. stays a real mismatch.""" + + def test_remix_vs_original_rejected_at_high_confidence(self): + assert is_acceptable_version_mismatch( + 'remix', 'original', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_instrumental_vs_original_rejected_at_high_confidence(self): + # The exact case test_acoustid_version_mismatch.py: + # test_instrumental_returned_for_vocal_request_fails pins — + # vocal asked, instrumental returned, must FAIL. + assert is_acceptable_version_mismatch( + 'instrumental', 'original', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_original_vs_instrumental_rejected_at_high_confidence(self): + # Reverse direction: caller asked for vocal, file is + # instrumental. + assert is_acceptable_version_mismatch( + 'original', 'instrumental', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_acoustic_vs_original_rejected_at_high_confidence(self): + assert is_acceptable_version_mismatch( + 'acoustic', 'original', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_demo_vs_original_rejected(self): + assert is_acceptable_version_mismatch( + 'demo', 'original', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + +class TestTwoSidedMismatchStaysStrict: + """Both sides have version markers but they disagree. Real + different-recording mismatch — must reject regardless of how + high the other scores are.""" + + def test_live_vs_remix_rejected_even_at_max(self): + assert is_acceptable_version_mismatch( + 'live', 'remix', + fingerprint_score=1.0, + title_similarity=1.0, + artist_similarity=1.0, + ) is False + + def test_acoustic_vs_instrumental_rejected(self): + assert is_acceptable_version_mismatch( + 'acoustic', 'instrumental', + fingerprint_score=0.99, + title_similarity=0.99, + artist_similarity=0.99, + ) is False + + def test_live_vs_acoustic_rejected(self): + assert is_acceptable_version_mismatch( + 'live', 'acoustic', + fingerprint_score=0.95, + title_similarity=0.90, + artist_similarity=1.0, + ) is False + + +class TestThresholdGates: + """One-sided + bare but one of the supporting signals is too weak. + Reject — fall through to FAIL.""" + + def test_low_fingerprint_score_rejected(self): + # Fingerprint score below threshold. Don't trust it enough. + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.50, + title_similarity=0.95, + artist_similarity=1.0, + ) is False + + def test_low_title_similarity_rejected(self): + # Bare titles disagree → different songs, not just MB metadata gap. + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.30, + artist_similarity=1.0, + ) is False + + def test_low_artist_similarity_rejected(self): + # Wrong artist — definitely not the same recording. + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.95, + artist_similarity=0.20, + ) is False + + def test_just_below_score_threshold_rejected(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.849, # default threshold 0.85 + title_similarity=0.95, + artist_similarity=1.0, + ) is False + + def test_just_below_title_threshold_rejected(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.699, # default threshold 0.70 + artist_similarity=1.0, + ) is False + + def test_just_below_artist_threshold_rejected(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.95, + title_similarity=0.95, + artist_similarity=0.599, # default threshold 0.60 + ) is False + + +class TestCustomThresholds: + def test_custom_score_threshold_accepts_when_loosened(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.70, + title_similarity=0.95, + artist_similarity=1.0, + score_threshold=0.65, + ) is True + + def test_custom_score_threshold_rejects_when_tightened(self): + assert is_acceptable_version_mismatch( + 'live', 'original', + fingerprint_score=0.90, + title_similarity=0.95, + artist_similarity=1.0, + score_threshold=0.95, + ) is False diff --git a/tests/metadata/test_artist_source_lookup.py b/tests/metadata/test_artist_source_lookup.py index 75ca4582..32c03cb6 100644 --- a/tests/metadata/test_artist_source_lookup.py +++ b/tests/metadata/test_artist_source_lookup.py @@ -31,6 +31,7 @@ EXPECTED_SOURCE_ID_FIELD = { "discogs": "discogs_id", "hydrabase": "soul_id", "musicbrainz": "musicbrainz_id", + "amazon": "amazon_id", } diff --git a/tests/test_download_plugin_conformance.py b/tests/test_download_plugin_conformance.py index 2b0f4e9d..4f380076 100644 --- a/tests/test_download_plugin_conformance.py +++ b/tests/test_download_plugin_conformance.py @@ -59,6 +59,7 @@ def _import_plugin_classes(): from core.deezer_download_client import DeezerDownloadClient from core.lidarr_download_client import LidarrDownloadClient from core.soundcloud_client import SoundcloudClient + from core.amazon_download_client import AmazonDownloadClient return { 'soulseek': SoulseekClient, @@ -69,10 +70,11 @@ def _import_plugin_classes(): 'deezer': DeezerDownloadClient, 'lidarr': LidarrDownloadClient, 'soundcloud': SoundcloudClient, + 'amazon': AmazonDownloadClient, } -def test_default_registry_registers_all_eight_sources(): +def test_default_registry_registers_all_sources(): """Smoke check that the foundation registry knows about every source the orchestrator historically dispatched to. If someone drops a registration here, every other test in this module would @@ -82,7 +84,7 @@ def test_default_registry_registers_all_eight_sources(): registry = build_default_registry() expected = { 'soulseek', 'youtube', 'tidal', 'qobuz', - 'hifi', 'deezer', 'lidarr', 'soundcloud', + 'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon', } assert set(registry.names()) == expected @@ -102,7 +104,7 @@ def test_deezer_dl_alias_is_registered_against_deezer_spec(): @pytest.mark.parametrize('plugin_name', [ 'soulseek', 'youtube', 'tidal', 'qobuz', - 'hifi', 'deezer', 'lidarr', 'soundcloud', + 'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon', ]) def test_plugin_class_has_all_required_methods(plugin_name): """Every registered plugin class exposes every protocol method @@ -122,7 +124,7 @@ def test_plugin_class_has_all_required_methods(plugin_name): @pytest.mark.parametrize('plugin_name', [ 'soulseek', 'youtube', 'tidal', 'qobuz', - 'hifi', 'deezer', 'lidarr', 'soundcloud', + 'hifi', 'deezer', 'lidarr', 'soundcloud', 'amazon', ]) def test_plugin_class_async_methods_are_coroutines(plugin_name): """Methods declared async in the protocol must be async on every diff --git a/tests/test_personalized_api.py b/tests/test_personalized_api.py new file mode 100644 index 00000000..7b7ef613 --- /dev/null +++ b/tests/test_personalized_api.py @@ -0,0 +1,181 @@ +"""Boundary tests for `core.personalized.api` handler functions. + +These are pure-function dispatchers — they take a manager + ids, +return a JSON-serializable dict. No Flask required, no real DB. +The Flask wiring in `web_server.py` adds `jsonify` + URL routing +on top. +""" + +from __future__ import annotations + +import sqlite3 +from types import SimpleNamespace +from typing import Any, List + +import pytest + +from core.personalized import api as _api +from core.personalized.manager import PersonalizedPlaylistManager +from core.personalized.specs import PlaylistKindRegistry, PlaylistKindSpec +from core.personalized.types import PlaylistConfig, Track +from database.personalized_schema import ensure_personalized_schema + + +class _FakeDB: + def __init__(self, path): + self.path = path + + def _get_connection(self): + c = sqlite3.connect(self.path) + c.row_factory = sqlite3.Row + return c + + +@pytest.fixture +def db(tmp_path): + p = str(tmp_path / 't.db') + conn = sqlite3.connect(p) + ensure_personalized_schema(conn) + conn.commit() + conn.close() + return _FakeDB(p) + + +@pytest.fixture +def registry(): + return PlaylistKindRegistry() + + +@pytest.fixture +def manager(db, registry): + return PersonalizedPlaylistManager(db, deps=None, registry=registry) + + +def _register(reg, kind='hidden_gems', requires_variant=False, generator=None): + spec = PlaylistKindSpec( + kind=kind, name_template=kind.replace('_', ' ').title(), + description=f'Description for {kind}', + default_config=PlaylistConfig(limit=20), + generator=generator or (lambda *a, **k: []), + requires_variant=requires_variant, + tags=['test'], + ) + reg.register(spec) + return spec + + +# ─── list_kinds ────────────────────────────────────────────────────── + + +class TestListKinds: + def test_lists_every_registered_kind(self, registry): + _register(registry, kind='hidden_gems') + _register(registry, kind='time_machine', requires_variant=True) + out = _api.list_kinds(registry) + assert out['success'] is True + kinds = {k['kind'] for k in out['kinds']} + assert kinds == {'hidden_gems', 'time_machine'} + + def test_kind_metadata_shape(self, registry): + _register(registry, kind='hidden_gems') + out = _api.list_kinds(registry) + kind = out['kinds'][0] + assert kind['kind'] == 'hidden_gems' + assert kind['requires_variant'] is False + assert kind['tags'] == ['test'] + assert kind['default_config']['limit'] == 20 + + def test_empty_registry(self): + out = _api.list_kinds(PlaylistKindRegistry()) + assert out == {'success': True, 'kinds': []} + + +# ─── list_playlists ───────────────────────────────────────────────── + + +class TestListPlaylists: + def test_returns_empty_when_no_playlists(self, manager, registry): + _register(registry) + out = _api.list_playlists(manager, profile_id=1) + assert out == {'success': True, 'playlists': []} + + def test_serializes_playlist_record(self, manager, registry): + _register(registry) + manager.ensure_playlist('hidden_gems', '', 1) + out = _api.list_playlists(manager, profile_id=1) + assert out['success'] is True + assert len(out['playlists']) == 1 + pl = out['playlists'][0] + assert pl['kind'] == 'hidden_gems' + assert pl['variant'] == '' + assert pl['name'] == 'Hidden Gems' + assert pl['track_count'] == 0 + assert pl['config']['limit'] == 20 + + +# ─── get_playlist_with_tracks ─────────────────────────────────────── + + +class TestGetPlaylistWithTracks: + def test_auto_creates_on_first_get(self, manager, registry): + _register(registry) + out = _api.get_playlist_with_tracks(manager, 'hidden_gems', '', 1) + assert out['success'] is True + assert out['playlist']['kind'] == 'hidden_gems' + assert out['tracks'] == [] + + def test_returns_persisted_tracks(self, manager, registry): + gen_calls = [] + + def gen(deps, variant, config): + gen_calls.append(1) + return [Track(track_name='X', artist_name='Y', spotify_track_id='sp-1')] + + _register(registry, generator=gen) + manager.refresh_playlist('hidden_gems', '', 1) + out = _api.get_playlist_with_tracks(manager, 'hidden_gems', '', 1) + assert len(out['tracks']) == 1 + assert out['tracks'][0]['track_name'] == 'X' + assert out['tracks'][0]['spotify_track_id'] == 'sp-1' + + def test_unknown_kind_raises_value_error(self, manager): + with pytest.raises(ValueError): + _api.get_playlist_with_tracks(manager, 'nope', '', 1) + + +# ─── refresh_playlist ─────────────────────────────────────────────── + + +class TestRefreshPlaylist: + def test_refresh_runs_generator_and_returns_tracks(self, manager, registry): + _register(registry, generator=lambda *a, **k: [ + Track(track_name='T1', artist_name='A', spotify_track_id='1'), + Track(track_name='T2', artist_name='B', spotify_track_id='2'), + ]) + out = _api.refresh_playlist(manager, 'hidden_gems', '', 1) + assert out['success'] is True + assert len(out['tracks']) == 2 + assert out['playlist']['track_count'] == 2 + assert out['playlist']['last_generated_at'] is not None + + def test_config_overrides_passed_through(self, manager, registry): + captured = {} + + def gen(deps, variant, config): + captured['limit'] = config.limit + return [] + + _register(registry, generator=gen) + _api.refresh_playlist(manager, 'hidden_gems', '', 1, config_overrides={'limit': 99}) + assert captured['limit'] == 99 + + +# ─── update_config ────────────────────────────────────────────────── + + +class TestUpdateConfig: + def test_patches_config(self, manager, registry): + _register(registry) + out = _api.update_config(manager, 'hidden_gems', '', 1, {'limit': 75}) + assert out['success'] is True + assert out['playlist']['config']['limit'] == 75 diff --git a/tests/test_personalized_generators_curated.py b/tests/test_personalized_generators_curated.py new file mode 100644 index 00000000..f8df2c33 --- /dev/null +++ b/tests/test_personalized_generators_curated.py @@ -0,0 +1,316 @@ +"""Boundary tests for the curated / hybrid personalized generators +(`daily_mix`, `fresh_tape`, `archives`, `seasonal_mix`).""" + +from __future__ import annotations + +import sqlite3 +from types import SimpleNamespace +from typing import Any, List +from unittest.mock import MagicMock + +import pytest + +from core.personalized.generators import archives as _arch_mod +from core.personalized.generators import daily_mix as _dm_mod +from core.personalized.generators import fresh_tape as _ft_mod +from core.personalized.generators import seasonal_mix as _sm_mod +from core.personalized.specs import get_registry +from core.personalized.types import PlaylistConfig + + +# ─── daily_mix ─────────────────────────────────────────────────────── + + +class _DailyMixService: + """Stub PersonalizedPlaylistsService for daily_mix tests.""" + + GENRE_MAPPING = {} + + def __init__(self, top_genres=None, genre_tracks=None): + self._top = top_genres or [] + self._tracks = genre_tracks or {} + self.calls: List[dict] = [] + + def get_top_genres_from_library(self, limit): + self.calls.append({'method': 'get_top_genres_from_library', 'limit': limit}) + return self._top + + def get_genre_playlist(self, genre, limit, **kw): + self.calls.append({'method': 'get_genre_playlist', 'genre': genre, 'limit': limit}) + return self._tracks.get(genre, []) + + +class TestDailyMix: + def test_registered(self): + spec = get_registry().get('daily_mix') + assert spec is not None + assert spec.requires_variant is True + + def test_variant_resolver_returns_ranks(self): + spec = get_registry().get('daily_mix') + ranks = spec.variant_resolver(SimpleNamespace(service=_DailyMixService())) + assert ranks == ['1', '2', '3', '4'] + + def test_resolves_rank_to_top_genre(self): + svc = _DailyMixService( + top_genres=[('Rock', 100), ('Pop', 80), ('Jazz', 30)], + genre_tracks={'Rock': [{'track_name': 'R', 'artist_name': 'A'}]}, + ) + out = _dm_mod.generate(SimpleNamespace(service=svc), '1', PlaylistConfig(limit=10)) + assert len(out) == 1 + assert out[0].track_name == 'R' + # Service called for top-genre lookup + genre playlist. + assert {c['method'] for c in svc.calls} == { + 'get_top_genres_from_library', 'get_genre_playlist', + } + + def test_rank_beyond_top_returns_empty(self): + svc = _DailyMixService(top_genres=[('Rock', 100)]) # only 1 top genre + out = _dm_mod.generate(SimpleNamespace(service=svc), '4', PlaylistConfig()) + assert out == [] + + def test_invalid_variant_raises(self): + deps = SimpleNamespace(service=_DailyMixService()) + with pytest.raises(ValueError, match='must be a rank int'): + _dm_mod.generate(deps, 'abc', PlaylistConfig()) + + +# ─── fresh_tape / archives shared shape ───────────────────────────── + + +class _StubPoolTrack: + def __init__(self, sid, name='T', artist='A', source='spotify'): + self.spotify_track_id = sid + self.itunes_track_id = None + self.deezer_track_id = None + self.track_name = name + self.artist_name = artist + self.album_name = 'Album' + self.album_cover_url = None + self.duration_ms = 200000 + self.popularity = 50 + self.track_data_json = None + self.source = source + + +class _CuratedDB: + def __init__(self, curated_ids=None, pool_tracks=None): + self.curated_ids = curated_ids or [] + self.pool_tracks = pool_tracks or [] + self.requested_keys: List[str] = [] + + def get_curated_playlist(self, key, profile_id=1): + self.requested_keys.append(key) + return list(self.curated_ids) + + def get_discovery_pool_tracks(self, **kwargs): + return list(self.pool_tracks) + + +def _curated_deps(db): + return SimpleNamespace( + database=db, + get_current_profile_id=lambda: 1, + get_active_discovery_source=lambda: 'spotify', + ) + + +class TestFreshTape: + def test_registered(self): + spec = get_registry().get('fresh_tape') + assert spec is not None + assert spec.requires_variant is False + assert spec.display_name('') == 'Fresh Tape' + + def test_returns_empty_when_no_curated_ids(self): + db = _CuratedDB(curated_ids=[]) + out = _ft_mod.generate(_curated_deps(db), '', PlaylistConfig()) + assert out == [] + + def test_hydrates_curated_ids_from_pool(self): + db = _CuratedDB( + curated_ids=['sp-1', 'sp-2', 'sp-missing'], + pool_tracks=[ + _StubPoolTrack('sp-1', name='Song1', artist='Artist1'), + _StubPoolTrack('sp-2', name='Song2', artist='Artist2'), + ], + ) + out = _ft_mod.generate(_curated_deps(db), '', PlaylistConfig()) + # Missing IDs silently skipped; order preserved. + assert [t.track_name for t in out] == ['Song1', 'Song2'] + + def test_tries_source_specific_then_fallback_key(self): + # First lookup (source-specific) returns []; second (generic) returns IDs. + class _DB: + def __init__(self): + self.calls = [] + self.responses = { + 'release_radar_spotify': [], + 'release_radar': ['sp-1'], + } + + def get_curated_playlist(self, key, profile_id=1): + self.calls.append(key) + return self.responses.get(key, []) + + def get_discovery_pool_tracks(self, **kw): + return [_StubPoolTrack('sp-1', name='Hit')] + + db = _DB() + out = _ft_mod.generate(_curated_deps(db), '', PlaylistConfig()) + assert db.calls == ['release_radar_spotify', 'release_radar'] + assert len(out) == 1 + + def test_respects_limit(self): + db = _CuratedDB( + curated_ids=[f'sp-{i}' for i in range(20)], + pool_tracks=[_StubPoolTrack(f'sp-{i}', name=f'T{i}') for i in range(20)], + ) + out = _ft_mod.generate(_curated_deps(db), '', PlaylistConfig(limit=5)) + assert len(out) == 5 + + def test_missing_database_dep_raises(self): + with pytest.raises(RuntimeError, match='missing `database`'): + _ft_mod.generate(SimpleNamespace(), '', PlaylistConfig()) + + +class TestArchives: + def test_registered(self): + spec = get_registry().get('archives') + assert spec is not None + assert spec.display_name('') == 'The Archives' + + def test_uses_discovery_weekly_curated_key(self): + db = _CuratedDB( + curated_ids=['sp-1'], + pool_tracks=[_StubPoolTrack('sp-1', name='Discover')], + ) + _arch_mod.generate(_curated_deps(db), '', PlaylistConfig()) + # Source-specific request fires first; fallback only fires + # when source-specific returns empty. Stub returns IDs on + # every call, so only the first key gets queried. + assert db.requested_keys[0] == 'discovery_weekly_spotify' + + +# ─── seasonal_mix ─────────────────────────────────────────────────── + + +class _SeasonalService: + def __init__(self, track_ids): + self.track_ids = track_ids + + def get_curated_seasonal_playlist(self, season_key, source=None): + return list(self.track_ids) + + +@pytest.fixture +def seasonal_db(tmp_path): + """Real sqlite DB with seasonal_tracks rows for hydration.""" + p = str(tmp_path / 'seasonal.db') + conn = sqlite3.connect(p) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute(""" + CREATE TABLE seasonal_tracks ( + id INTEGER PRIMARY KEY, + season_key TEXT, source TEXT, + spotify_track_id TEXT, track_name TEXT, artist_name TEXT, + album_name TEXT, album_cover_url TEXT, + duration_ms INTEGER, popularity INTEGER, track_data_json TEXT + ) + """) + seed = [ + ('halloween', 'spotify', 'sp-1', 'Spooky', 'Ghost Band', 'Album1', None, 200000, 80, '{"id":"sp-1"}'), + ('halloween', 'spotify', 'sp-2', 'Haunted', 'Ghost Band', 'Album2', None, 210000, 70, None), + ('halloween', 'spotify', 'sp-extra', 'Extra', 'Other', 'Album3', None, 200000, 60, None), + ] + cursor.executemany(""" + INSERT INTO seasonal_tracks + (season_key, source, spotify_track_id, track_name, artist_name, + album_name, album_cover_url, duration_ms, popularity, track_data_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, seed) + conn.commit() + conn.close() + + class _DB: + def __init__(self, path): self.path = path + def _get_connection(self): + c = sqlite3.connect(self.path) + c.row_factory = sqlite3.Row + return c + return _DB(p) + + +class TestSeasonalMix: + def test_registered(self): + spec = get_registry().get('seasonal_mix') + assert spec is not None + assert spec.requires_variant is True + + def test_variant_resolver_returns_seasons(self): + spec = get_registry().get('seasonal_mix') + seasons = spec.variant_resolver(None) + assert 'halloween' in seasons + assert 'christmas' in seasons + + def test_no_variant_raises(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-1']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + with pytest.raises(ValueError, match='requires a season variant'): + _sm_mod.generate(deps, '', PlaylistConfig()) + + def test_hydrates_curated_ids_in_order(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-2', 'sp-1']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig()) + assert [t.track_name for t in out] == ['Haunted', 'Spooky'] + + def test_missing_track_id_silently_skipped(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-1', 'sp-not-in-db']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig()) + assert len(out) == 1 + assert out[0].track_name == 'Spooky' + + def test_track_data_json_round_trips(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-1']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig()) + # sp-1 had JSON; sp-2 had None. + assert out[0].track_data_json == {'id': 'sp-1'} + + def test_empty_curated_returns_empty(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService([]), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig()) + assert out == [] + + def test_respects_limit(self, seasonal_db): + deps = SimpleNamespace( + seasonal_service=_SeasonalService(['sp-1', 'sp-2', 'sp-extra']), + database=seasonal_db, + get_active_discovery_source=lambda: 'spotify', + ) + out = _sm_mod.generate(deps, 'halloween', PlaylistConfig(limit=2)) + assert len(out) == 2 + + def test_missing_seasonal_service_raises(self): + deps = SimpleNamespace(database=object()) + with pytest.raises(RuntimeError, match='missing `seasonal_service`'): + _sm_mod.generate(deps, 'halloween', PlaylistConfig()) diff --git a/tests/test_personalized_generators_singletons.py b/tests/test_personalized_generators_singletons.py new file mode 100644 index 00000000..6f39fd5a --- /dev/null +++ b/tests/test_personalized_generators_singletons.py @@ -0,0 +1,147 @@ +"""Boundary tests for the singleton-kind personalized generators +(`hidden_gems`, `discovery_shuffle`, `popular_picks`). + +Each generator wraps the legacy +``PersonalizedPlaylistsService`` method 1:1, so the tests pin: +- registration side-effect at import +- generator forwards `config.limit` correctly +- empty / None / non-dict service output → [] +- tracks coerced through `Track.from_dict` +- missing service in deps raises a clear error""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any, List + +import pytest + +# Importing each generator triggers registration as a side-effect. +from core.personalized.generators import discovery_shuffle as _ds_mod +from core.personalized.generators import hidden_gems as _hg_mod +from core.personalized.generators import popular_picks as _pp_mod +from core.personalized.specs import get_registry +from core.personalized.types import PlaylistConfig + + +class _StubService: + """Records every call so tests can assert on dispatched limits.""" + + def __init__(self, return_value=None): + self.calls: List[dict] = [] + self.return_value = return_value if return_value is not None else [] + + def get_hidden_gems(self, limit): + self.calls.append({'method': 'get_hidden_gems', 'limit': limit}) + return self.return_value + + def get_discovery_shuffle(self, limit): + self.calls.append({'method': 'get_discovery_shuffle', 'limit': limit}) + return self.return_value + + def get_popular_picks(self, limit): + self.calls.append({'method': 'get_popular_picks', 'limit': limit}) + return self.return_value + + +def _deps(svc): + return SimpleNamespace(service=svc) + + +# ─── registration ──────────────────────────────────────────────────── + + +class TestRegistration: + def test_hidden_gems_registered(self): + spec = get_registry().get('hidden_gems') + assert spec is not None + assert spec.kind == 'hidden_gems' + assert spec.requires_variant is False + assert spec.default_config.limit == 50 + + def test_discovery_shuffle_registered(self): + spec = get_registry().get('discovery_shuffle') + assert spec is not None + assert spec.requires_variant is False + + def test_popular_picks_registered(self): + spec = get_registry().get('popular_picks') + assert spec is not None + assert spec.requires_variant is False + + def test_display_names(self): + assert get_registry().get('hidden_gems').display_name('') == 'Hidden Gems' + assert get_registry().get('discovery_shuffle').display_name('') == 'Discovery Shuffle' + assert get_registry().get('popular_picks').display_name('') == 'Popular Picks' + + +# ─── generator dispatch ────────────────────────────────────────────── + + +class TestHiddenGemsGenerator: + def test_forwards_limit(self): + svc = _StubService() + _hg_mod.generate(_deps(svc), '', PlaylistConfig(limit=75)) + assert svc.calls == [{'method': 'get_hidden_gems', 'limit': 75}] + + def test_uses_default_limit_when_config_default(self): + svc = _StubService() + _hg_mod.generate(_deps(svc), '', PlaylistConfig()) + assert svc.calls[0]['limit'] == 50 + + def test_coerces_tracks(self): + svc = _StubService(return_value=[ + {'track_name': 'A', 'artist_name': 'X', 'spotify_track_id': 'sp-1'}, + {'track_name': 'B', 'artist_name': 'Y', 'spotify_track_id': 'sp-2'}, + ]) + out = _hg_mod.generate(_deps(svc), '', PlaylistConfig()) + assert len(out) == 2 + assert out[0].track_name == 'A' + assert out[0].spotify_track_id == 'sp-1' + + def test_empty_service_output_returns_empty_list(self): + svc = _StubService(return_value=[]) + out = _hg_mod.generate(_deps(svc), '', PlaylistConfig()) + assert out == [] + + def test_none_service_output_returns_empty_list(self): + svc = _StubService(return_value=None) + out = _hg_mod.generate(_deps(svc), '', PlaylistConfig()) + assert out == [] + + +class TestDiscoveryShuffleGenerator: + def test_forwards_limit(self): + svc = _StubService() + _ds_mod.generate(_deps(svc), '', PlaylistConfig(limit=42)) + assert svc.calls == [{'method': 'get_discovery_shuffle', 'limit': 42}] + + def test_coerces_tracks(self): + svc = _StubService(return_value=[{'track_name': 'Z', 'artist_name': 'Q'}]) + out = _ds_mod.generate(_deps(svc), '', PlaylistConfig()) + assert out[0].track_name == 'Z' + + +class TestPopularPicksGenerator: + def test_forwards_limit(self): + svc = _StubService() + _pp_mod.generate(_deps(svc), '', PlaylistConfig(limit=10)) + assert svc.calls == [{'method': 'get_popular_picks', 'limit': 10}] + + +# ─── deps validation ───────────────────────────────────────────────── + + +class TestDepsValidation: + def test_missing_service_raises(self): + # No `service` attribute on deps. + deps = SimpleNamespace() + with pytest.raises(RuntimeError, match='missing `service`'): + _hg_mod.generate(deps, '', PlaylistConfig()) + + def test_dict_form_deps_accepted(self): + # generators._common.get_service tolerates dict deps too. + svc = _StubService() + out = _hg_mod.generate({'service': svc}, '', PlaylistConfig()) + assert isinstance(out, list) + assert svc.calls diff --git a/tests/test_personalized_generators_variants.py b/tests/test_personalized_generators_variants.py new file mode 100644 index 00000000..40eb04d6 --- /dev/null +++ b/tests/test_personalized_generators_variants.py @@ -0,0 +1,131 @@ +"""Boundary tests for variant-bearing personalized generators +(`time_machine` per decade, `genre_playlist` per genre). + +Each generator coerces a URL-safe variant string into the form the +legacy service expects, then forwards. Tests pin the variant +parsing + service dispatch + variant_resolver listing.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import List + +import pytest + +from core.personalized.generators import genre_playlist as _gp_mod +from core.personalized.generators import time_machine as _tm_mod +from core.personalized.specs import get_registry +from core.personalized.types import PlaylistConfig + + +class _StubService: + GENRE_MAPPING = { + 'Electronic/Dance': ['house', 'techno'], + 'Hip Hop/Rap': ['hip hop', 'rap'], + 'Rock': ['rock', 'punk'], + } + + def __init__(self): + self.calls: List[dict] = [] + + def get_decade_playlist(self, decade, limit, **kw): + self.calls.append({'method': 'get_decade_playlist', 'decade': decade, 'limit': limit}) + return [{'track_name': f'D{decade}', 'artist_name': 'A'}] + + def get_genre_playlist(self, genre, limit, **kw): + self.calls.append({'method': 'get_genre_playlist', 'genre': genre, 'limit': limit}) + return [{'track_name': f'G{genre}', 'artist_name': 'A'}] + + +def _deps(): + return SimpleNamespace(service=_StubService()) + + +# ─── time_machine ─────────────────────────────────────────────────── + + +class TestTimeMachine: + def test_registered(self): + spec = get_registry().get('time_machine') + assert spec is not None + assert spec.requires_variant is True + assert spec.variant_resolver is not None + + def test_variant_resolver_returns_decades(self): + spec = get_registry().get('time_machine') + decades = spec.variant_resolver(_deps()) + assert '1980s' in decades + assert '2020s' in decades + # All decades should be 4-digit + 's' + for d in decades: + assert d.endswith('s') + assert d[:-1].isdigit() + + def test_decade_label_to_year(self): + deps = _deps() + _tm_mod.generate(deps, '1980s', PlaylistConfig(limit=20)) + assert deps.service.calls == [ + {'method': 'get_decade_playlist', 'decade': 1980, 'limit': 20} + ] + + def test_invalid_variant_raises(self): + deps = _deps() + with pytest.raises(ValueError, match='not a decade label'): + _tm_mod.generate(deps, 'banana', PlaylistConfig()) + + def test_out_of_range_year_raises(self): + deps = _deps() + with pytest.raises(ValueError, match='out of range'): + _tm_mod.generate(deps, '1500s', PlaylistConfig()) + + def test_tolerates_no_s_suffix(self): + deps = _deps() + _tm_mod.generate(deps, '1990', PlaylistConfig()) + assert deps.service.calls[0]['decade'] == 1990 + + def test_default_limit_is_100(self): + spec = get_registry().get('time_machine') + assert spec.default_config.limit == 100 + + def test_display_name_with_variant(self): + spec = get_registry().get('time_machine') + assert spec.display_name('1980s') == 'Time Machine — 1980s' + + +# ─── genre_playlist ───────────────────────────────────────────────── + + +class TestGenrePlaylist: + def test_registered(self): + spec = get_registry().get('genre_playlist') + assert spec is not None + assert spec.requires_variant is True + + def test_variant_resolver_normalizes_parent_keys(self): + spec = get_registry().get('genre_playlist') + variants = spec.variant_resolver(_deps()) + # 'Electronic/Dance' → 'electronic_dance' (slash → underscore + lowercase) + assert 'electronic_dance' in variants + assert 'hip_hop_rap' in variants + assert 'rock' in variants + + def test_normalized_variant_resolves_to_parent_key(self): + deps = _deps() + _gp_mod.generate(deps, 'electronic_dance', PlaylistConfig()) + # Service receives ORIGINAL parent key. + assert deps.service.calls[0]['genre'] == 'Electronic/Dance' + + def test_unknown_variant_passed_through_as_freeform(self): + # Service handles partial-matching for free-form keywords. + deps = _deps() + _gp_mod.generate(deps, 'shoegaze', PlaylistConfig()) + assert deps.service.calls[0]['genre'] == 'shoegaze' + + def test_empty_variant_raises(self): + deps = _deps() + with pytest.raises(ValueError, match='requires a variant'): + _gp_mod.generate(deps, '', PlaylistConfig()) + + def test_display_name(self): + spec = get_registry().get('genre_playlist') + assert spec.display_name('electronic_dance') == 'Genre — electronic_dance' diff --git a/tests/test_personalized_manager.py b/tests/test_personalized_manager.py new file mode 100644 index 00000000..6b12ca7a --- /dev/null +++ b/tests/test_personalized_manager.py @@ -0,0 +1,532 @@ +"""Boundary tests for the personalized-playlists foundation +(``core.personalized.types`` + ``core.personalized.specs`` + +``core.personalized.manager``). + +Pin every shape the storage layer + lifecycle has to handle so the +generators that arrive in subsequent commits can rely on a stable +contract: ensure_playlist auto-creates from default config, refresh +atomically replaces the snapshot + appends history, generator +exceptions don't lose the previous good snapshot, config patches +preserve unsent fields, recent_track_ids honors the day window, +list_playlists orders newest-first.""" + +from __future__ import annotations + +import os +import sqlite3 +import tempfile +from typing import Any, List +from unittest.mock import MagicMock + +import pytest + +from core.personalized.manager import PersonalizedPlaylistManager +from core.personalized.specs import PlaylistKindRegistry, PlaylistKindSpec +from core.personalized.types import PlaylistConfig, Track +from database.personalized_schema import ensure_personalized_schema + + +# ─── shared fixtures ───────────────────────────────────────────────── + + +class _FakeDB: + """Minimal MusicDatabase stand-in — gives the manager a real + sqlite connection so the manager exercises actual SQL.""" + + def __init__(self, path: str): + self.path = path + + def _get_connection(self): + conn = sqlite3.connect(self.path) + conn.row_factory = sqlite3.Row + return conn + + +@pytest.fixture +def db_path(tmp_path): + p = str(tmp_path / 'test.db') + conn = sqlite3.connect(p) + ensure_personalized_schema(conn) + conn.commit() + conn.close() + return p + + +@pytest.fixture +def db(db_path): + return _FakeDB(db_path) + + +@pytest.fixture +def registry(): + r = PlaylistKindRegistry() + return r + + +def _make_track(name='T1', artist='A1', sid='spot-1', source='spotify') -> Track: + return Track( + track_name=name, artist_name=artist, album_name='Album', + spotify_track_id=sid, source=source, + duration_ms=200000, popularity=50, + ) + + +# ─── PlaylistConfig ────────────────────────────────────────────────── + + +class TestPlaylistConfig: + def test_default_values(self): + c = PlaylistConfig() + assert c.limit == 50 + assert c.max_per_album == 2 + assert c.max_per_artist == 3 + assert c.popularity_min is None + assert c.popularity_max is None + assert c.exclude_recent_days == 0 + assert c.recency_days is None + assert c.seed is None + assert c.extra == {} + + def test_round_trip_through_json_dict(self): + c = PlaylistConfig( + limit=100, max_per_album=5, max_per_artist=10, + popularity_min=20, popularity_max=80, + exclude_recent_days=14, recency_days=180, + seed=42, extra={'selected_seasons': ['halloween', 'christmas']}, + ) + d = c.to_json_dict() + c2 = PlaylistConfig.from_json_dict(d) + assert c2 == c + + def test_from_json_dict_handles_none(self): + c = PlaylistConfig.from_json_dict(None) + assert c == PlaylistConfig() + + def test_from_json_dict_handles_non_dict(self): + c = PlaylistConfig.from_json_dict('garbage') # type: ignore + assert c == PlaylistConfig() + + def test_from_json_dict_missing_fields_use_defaults(self): + c = PlaylistConfig.from_json_dict({'limit': 75}) + assert c.limit == 75 + assert c.max_per_album == 2 # default + + def test_merged_overrides_only_named_fields(self): + base = PlaylistConfig(limit=50, popularity_min=20) + out = base.merged({'limit': 100}) + assert out.limit == 100 + assert out.popularity_min == 20 # untouched + + def test_merged_extra_dict_is_deep_merged(self): + base = PlaylistConfig(extra={'a': 1, 'b': 2}) + out = base.merged({'extra': {'b': 99, 'c': 3}}) + assert out.extra == {'a': 1, 'b': 99, 'c': 3} + + def test_merged_ignores_unknown_keys(self): + base = PlaylistConfig() + out = base.merged({'unknown_field': 'foo'}) + assert out == base + + +# ─── Track ──────────────────────────────────────────────────────────── + + +class TestTrack: + def test_from_dict_legacy_shape(self): + d = { + 'track_name': 'Song', 'artist_name': 'Band', + 'album_name': 'Album', 'spotify_track_id': 'spot-1', + 'duration_ms': 200000, 'popularity': 60, + '_artist_genres_raw': '["rock"]', # ignored extra + } + t = Track.from_dict(d) + assert t.track_name == 'Song' + assert t.spotify_track_id == 'spot-1' + assert t.duration_ms == 200000 + + def test_primary_id_prefers_spotify(self): + t = Track( + track_name='', artist_name='', + spotify_track_id='spot', itunes_track_id='itu', deezer_track_id='dee', + ) + assert t.primary_id() == 'spot' + + def test_primary_id_falls_back_through_sources(self): + t = Track(track_name='', artist_name='', itunes_track_id='itu') + assert t.primary_id() == 'itu' + t2 = Track(track_name='', artist_name='', deezer_track_id='dee') + assert t2.primary_id() == 'dee' + + def test_primary_id_none_when_no_sources(self): + t = Track(track_name='', artist_name='') + assert t.primary_id() is None + + +# ─── PlaylistKindRegistry ──────────────────────────────────────────── + + +class TestRegistry: + def test_register_and_get(self, registry): + spec = PlaylistKindSpec( + kind='hidden_gems', name_template='Hidden Gems', + description='', default_config=PlaylistConfig(), + generator=lambda *a, **k: [], + ) + registry.register(spec) + assert registry.get('hidden_gems') is spec + assert registry.get('nonexistent') is None + + def test_duplicate_registration_raises(self, registry): + spec = PlaylistKindSpec( + kind='x', name_template='X', description='', + default_config=PlaylistConfig(), generator=lambda *a, **k: [], + ) + registry.register(spec) + with pytest.raises(ValueError, match='already registered'): + registry.register(spec) + + def test_display_name_singleton(self): + spec = PlaylistKindSpec( + kind='x', name_template='Hidden Gems', description='', + default_config=PlaylistConfig(), generator=lambda *a, **k: [], + ) + assert spec.display_name('') == 'Hidden Gems' + + def test_display_name_with_variant(self): + spec = PlaylistKindSpec( + kind='x', name_template='Time Machine — {variant}', + description='', default_config=PlaylistConfig(), + generator=lambda *a, **k: [], + ) + assert spec.display_name('1980s') == 'Time Machine — 1980s' + + def test_kinds_listing(self, registry): + for k in ('a', 'b', 'c'): + registry.register(PlaylistKindSpec( + kind=k, name_template=k, description='', + default_config=PlaylistConfig(), generator=lambda *a, **k: [], + )) + assert set(registry.kinds()) == {'a', 'b', 'c'} + + +# ─── PersonalizedPlaylistManager ───────────────────────────────────── + + +def _register_simple_kind(registry, generator, kind='hidden_gems', requires_variant=False): + spec = PlaylistKindSpec( + kind=kind, name_template=kind.replace('_', ' ').title(), + description='', default_config=PlaylistConfig(limit=10), + generator=generator, requires_variant=requires_variant, + ) + registry.register(spec) + return spec + + +class TestEnsurePlaylist: + def test_creates_row_with_default_config(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.ensure_playlist('hidden_gems', '', 1) + assert record.id > 0 + assert record.kind == 'hidden_gems' + assert record.variant == '' + assert record.profile_id == 1 + assert record.config.limit == 10 # from default + assert record.track_count == 0 + assert record.last_generated_at is None + + def test_returns_same_row_on_second_call(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.ensure_playlist('hidden_gems', '', 1) + r2 = mgr.ensure_playlist('hidden_gems', '', 1) + assert r1.id == r2.id + + def test_variant_creates_separate_row(self, db, registry): + _register_simple_kind( + registry, lambda *a, **k: [], kind='time_machine', requires_variant=True, + ) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.ensure_playlist('time_machine', '1980s', 1) + r2 = mgr.ensure_playlist('time_machine', '1990s', 1) + assert r1.id != r2.id + + def test_unknown_kind_raises(self, db, registry): + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + with pytest.raises(ValueError, match='Unknown playlist kind'): + mgr.ensure_playlist('does_not_exist', '', 1) + + def test_required_variant_missing_raises(self, db, registry): + _register_simple_kind( + registry, lambda *a, **k: [], kind='time_machine', requires_variant=True, + ) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + with pytest.raises(ValueError, match='requires a variant'): + mgr.ensure_playlist('time_machine', '', 1) + + +class TestRefreshPlaylist: + def test_refresh_persists_tracks(self, db, registry): + tracks = [_make_track('S1', 'A1', 'sp1'), _make_track('S2', 'A1', 'sp2')] + _register_simple_kind(registry, lambda deps, variant, config: tracks) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.refresh_playlist('hidden_gems', '', 1) + assert record.track_count == 2 + assert record.last_generated_at is not None + assert record.last_generation_error is None + + persisted = mgr.get_playlist_tracks(record.id) + assert len(persisted) == 2 + assert persisted[0].track_name == 'S1' + assert persisted[1].track_name == 'S2' + + def test_refresh_replaces_previous_snapshot_atomically(self, db, registry): + run = {'tracks': [_make_track('first')]} + + def gen(deps, variant, config): + return run['tracks'] + + _register_simple_kind(registry, gen) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r1.track_count == 1 + + run['tracks'] = [_make_track('A'), _make_track('B'), _make_track('C')] + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r2.id == r1.id + assert r2.track_count == 3 + + persisted = mgr.get_playlist_tracks(r2.id) + assert [t.track_name for t in persisted] == ['A', 'B', 'C'] + + def test_generator_exception_preserves_previous_snapshot(self, db, registry): + run = {'mode': 'success'} + + def gen(deps, variant, config): + if run['mode'] == 'fail': + raise RuntimeError('generator boom') + return [_make_track('first')] + + _register_simple_kind(registry, gen) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r1.track_count == 1 + + run['mode'] = 'fail' + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + # Previous snapshot preserved. + assert r2.track_count == 1 + # Error stamped on row. + assert r2.last_generation_error is not None + assert 'generator boom' in r2.last_generation_error + # Tracks still queryable. + persisted = mgr.get_playlist_tracks(r2.id) + assert len(persisted) == 1 + + def test_config_overrides_passed_to_generator(self, db, registry): + captured = {} + + def gen(deps, variant, config): + captured['limit'] = config.limit + return [] + + _register_simple_kind(registry, gen) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1, config_overrides={'limit': 200}) + assert captured['limit'] == 200 + + def test_refresh_records_source_from_first_track(self, db, registry): + tracks = [_make_track(source='spotify'), _make_track(source='deezer')] + _register_simple_kind(registry, lambda *a, **k: tracks) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.refresh_playlist('hidden_gems', '', 1) + assert record.last_generation_source == 'spotify' + + def test_track_data_json_round_trips(self, db, registry): + nested = {'id': 'spot-1', 'name': 'Foo', 'artists': [{'name': 'Bar'}]} + track = Track( + track_name='Foo', artist_name='Bar', + spotify_track_id='spot-1', track_data_json=nested, + ) + _register_simple_kind(registry, lambda *a, **k: [track]) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.refresh_playlist('hidden_gems', '', 1) + persisted = mgr.get_playlist_tracks(record.id) + assert persisted[0].track_data_json == nested + + +class TestUpdateConfig: + def test_patch_merges_with_stored(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + record = mgr.update_config('hidden_gems', '', 1, {'limit': 75}) + assert record.config.limit == 75 + # Other fields kept. + assert record.config.max_per_album == 2 + + def test_patch_extra_dict_deep_merges(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.update_config('hidden_gems', '', 1, {'extra': {'a': 1}}) + record = mgr.update_config('hidden_gems', '', 1, {'extra': {'b': 2}}) + assert record.config.extra == {'a': 1, 'b': 2} + + +class TestListPlaylists: + def test_lists_all_playlists_for_profile(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [], kind='hidden_gems') + _register_simple_kind(registry, lambda *a, **k: [], kind='popular_picks') + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('popular_picks', '', 1) + records = mgr.list_playlists(1) + kinds = {r.kind for r in records} + assert kinds == {'hidden_gems', 'popular_picks'} + + def test_does_not_list_other_profiles(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('hidden_gems', '', 2) + assert len(mgr.list_playlists(1)) == 1 + assert len(mgr.list_playlists(2)) == 1 + + +class TestStalenessFilter: + """`config.exclude_recent_days > 0` drops tracks served by this + kind for this profile in the last N days.""" + + def test_zero_days_means_no_filter(self, db, registry): + # Default config has exclude_recent_days=0; everything passes. + tracks = [_make_track(sid='spot-1'), _make_track(sid='spot-2')] + run = {'tracks': tracks} + _register_simple_kind(registry, lambda *a, **k: run['tracks']) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + # Refresh again with same tracks — no filter, all should persist. + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r2.track_count == 2 + + def test_positive_days_filters_recently_served(self, db, registry): + run = {'tracks': [_make_track(sid='spot-1'), _make_track(sid='spot-2')]} + _register_simple_kind(registry, lambda *a, **k: run['tracks']) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r1.track_count == 2 + # Update config to exclude tracks served in last 7 days. + mgr.update_config('hidden_gems', '', 1, {'exclude_recent_days': 7}) + # Same generator output now → all tracks just got served, all filtered out. + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + assert r2.track_count == 0 + + def test_filter_preserves_non_recent_tracks(self, db, registry): + run = {'tracks': [_make_track(sid='spot-1')]} + _register_simple_kind(registry, lambda *a, **k: run['tracks']) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + r1 = mgr.refresh_playlist('hidden_gems', '', 1) + mgr.update_config('hidden_gems', '', 1, {'exclude_recent_days': 7}) + # New generator output with a NEW id — should pass. + run['tracks'] = [_make_track(sid='spot-1'), _make_track(sid='spot-NEW')] + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + # spot-1 was just served, dropped. spot-NEW is fresh, kept. + assert r2.track_count == 1 + persisted = mgr.get_playlist_tracks(r2.id) + assert persisted[0].spotify_track_id == 'spot-NEW' + + def test_tracks_without_primary_id_pass_through(self, db, registry): + # Track with no source IDs — primary_id() is None — staleness + # filter has nothing to dedupe on, so the track passes. + track_no_id = Track(track_name='X', artist_name='Y', source='spotify') + run = {'tracks': [track_no_id]} + _register_simple_kind(registry, lambda *a, **k: run['tracks']) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1) + mgr.update_config('hidden_gems', '', 1, {'exclude_recent_days': 7}) + r2 = mgr.refresh_playlist('hidden_gems', '', 1) + # Track is kept because there's no id to match against history. + assert r2.track_count == 1 + + +class TestStaleFlag: + """`is_stale` flips when upstream data changes; refresh clears it.""" + + def test_default_is_false(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + record = mgr.ensure_playlist('hidden_gems', '', 1) + assert record.is_stale is False + + def test_mark_kinds_stale_flips_flag(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [], kind='hidden_gems') + _register_simple_kind(registry, lambda *a, **k: [], kind='discovery_shuffle') + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('discovery_shuffle', '', 1) + n = mgr.mark_kinds_stale(['hidden_gems', 'discovery_shuffle']) + assert n == 2 + assert mgr.ensure_playlist('hidden_gems', '', 1).is_stale is True + assert mgr.ensure_playlist('discovery_shuffle', '', 1).is_stale is True + + def test_mark_kinds_stale_only_matching_kinds(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [], kind='hidden_gems') + _register_simple_kind(registry, lambda *a, **k: [], kind='popular_picks') + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('popular_picks', '', 1) + mgr.mark_kinds_stale(['hidden_gems']) + assert mgr.ensure_playlist('hidden_gems', '', 1).is_stale is True + assert mgr.ensure_playlist('popular_picks', '', 1).is_stale is False + + def test_mark_kinds_stale_scopes_to_profile(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.ensure_playlist('hidden_gems', '', 2) + mgr.mark_kinds_stale(['hidden_gems'], profile_id=1) + assert mgr.ensure_playlist('hidden_gems', '', 1).is_stale is True + assert mgr.ensure_playlist('hidden_gems', '', 2).is_stale is False + + def test_refresh_clears_stale_flag(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [_make_track()]) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + mgr.mark_kinds_stale(['hidden_gems']) + assert mgr.ensure_playlist('hidden_gems', '', 1).is_stale is True + record = mgr.refresh_playlist('hidden_gems', '', 1) + assert record.is_stale is False + + def test_mark_kinds_stale_empty_list_noop(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: []) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.ensure_playlist('hidden_gems', '', 1) + n = mgr.mark_kinds_stale([]) + assert n == 0 + + +class TestStalenessHistory: + def test_recent_track_ids_returns_zero_when_days_zero(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [_make_track(sid='spot-1')]) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1) + assert mgr.recent_track_ids(1, 'hidden_gems', 0) == [] + + def test_recent_track_ids_after_refresh(self, db, registry): + _register_simple_kind( + registry, + lambda *a, **k: [_make_track(sid='spot-1'), _make_track(sid='spot-2')], + ) + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1) + recent = mgr.recent_track_ids(1, 'hidden_gems', 7) + assert set(recent) == {'spot-1', 'spot-2'} + + def test_recent_track_ids_scoped_to_kind(self, db, registry): + _register_simple_kind(registry, lambda *a, **k: [_make_track(sid='gem-1')], kind='hidden_gems') + _register_simple_kind(registry, lambda *a, **k: [_make_track(sid='pop-1')], kind='popular_picks') + mgr = PersonalizedPlaylistManager(db, deps=None, registry=registry) + mgr.refresh_playlist('hidden_gems', '', 1) + mgr.refresh_playlist('popular_picks', '', 1) + assert mgr.recent_track_ids(1, 'hidden_gems', 7) == ['gem-1'] + assert mgr.recent_track_ids(1, 'popular_picks', 7) == ['pop-1'] diff --git a/tests/test_reorganize_tag_source.py b/tests/test_reorganize_tag_source.py new file mode 100644 index 00000000..9eee1ff5 --- /dev/null +++ b/tests/test_reorganize_tag_source.py @@ -0,0 +1,602 @@ +"""Boundary tests for ``core.library.reorganize_tag_source``. + +Pin every shape the embedded-tag → reorganize-context adapter has to +handle so future drift fails here instead of at runtime against a +real library: empty / missing essentials, multi-value vs single-string +artist tags, ID3-style ``"5/12"`` track-number values, year +normalization across date shapes, releasetype validation, multi-disc +parsing, defensive paths against bad input. + +The wrapper :func:`read_album_track_from_file` is tested against a +fake ``read_embedded_tags_fn`` so no real mutagen IO happens here. +""" + +from __future__ import annotations + +import sys +import types +from typing import Any, Dict + +import pytest + + +# ── stubs (other tests rely on these too — keep the shape consistent) ── +if 'utils.logging_config' not in sys.modules: + utils_mod = types.ModuleType('utils') + logging_mod = types.ModuleType('utils.logging_config') + logging_mod.get_logger = lambda name: type('L', (), { + 'debug': lambda *a, **k: None, + 'info': lambda *a, **k: None, + 'warning': lambda *a, **k: None, + 'error': lambda *a, **k: None, + })() + sys.modules['utils'] = utils_mod + sys.modules['utils.logging_config'] = logging_mod + + +from core.library.reorganize_tag_source import ( + extract_album_meta_from_tags, + extract_track_meta_from_tags, + read_album_track_from_file, + normalize_resolved_path, +) + + +# ─── extract_track_meta_from_tags ───────────────────────────────────── + + +class TestExtractTrackMeta: + def test_full_set_returns_canonical_shape(self): + out = extract_track_meta_from_tags({ + 'title': 'HUMBLE.', + 'artist': 'Kendrick Lamar', + 'tracknumber': '4', + 'discnumber': '1', + }) + assert out is not None + assert out['name'] == 'HUMBLE.' + assert out['title'] == 'HUMBLE.' + assert out['track_number'] == 4 + assert out['disc_number'] == 1 + assert out['artists'] == [{'name': 'Kendrick Lamar'}] + assert out['duration_ms'] == 0 + assert out['id'] == '' + assert out['uri'] == '' + + def test_missing_title_returns_none(self): + assert extract_track_meta_from_tags({'artist': 'foo'}) is None + assert extract_track_meta_from_tags({'title': '', 'artist': 'foo'}) is None + assert extract_track_meta_from_tags({'title': ' ', 'artist': 'foo'}) is None + + def test_missing_artist_returns_none(self): + assert extract_track_meta_from_tags({'title': 'Song'}) is None + assert extract_track_meta_from_tags({'title': 'Song', 'artist': ''}) is None + + def test_multi_value_artists_field_takes_precedence(self): + out = extract_track_meta_from_tags({ + 'title': 'Collab', + 'artist': 'Foo Bar', + 'artists': 'Foo, Bar, Baz', # multi-value tag joined by reader + }) + assert out is not None + assert out['artists'] == [{'name': 'Foo'}, {'name': 'Bar'}, {'name': 'Baz'}] + + def test_artist_string_split_on_known_separators(self): + for sep_input, expected in [ + ('Foo, Bar', ['Foo', 'Bar']), + ('Foo & Bar', ['Foo', 'Bar']), + ('Foo feat. Bar', ['Foo', 'Bar']), + ('Foo ft Bar', ['Foo', 'Bar']), + ('Foo featuring Bar', ['Foo', 'Bar']), + ('Foo / Bar', ['Foo', 'Bar']), + ('Foo; Bar', ['Foo', 'Bar']), + ('Foo x Bar', ['Foo', 'Bar']), + ('Foo with Bar', ['Foo', 'Bar']), + ]: + out = extract_track_meta_from_tags({'title': 't', 'artist': sep_input}) + assert out is not None + names = [a['name'] for a in out['artists']] + assert names == expected, f"failed for {sep_input!r}: got {names}" + + def test_artist_dedup_case_insensitive(self): + out = extract_track_meta_from_tags({ + 'title': 't', + 'artists': 'Foo, foo, FOO, Bar', + }) + assert out is not None + names = [a['name'] for a in out['artists']] + assert names == ['Foo', 'Bar'] + + def test_id3_track_total_shape(self): + # ID3 stores TRCK as "5/12" — caller must use the head only. + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': '5/12', + }) + assert out['track_number'] == 5 + + def test_disc_total_shape(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'discnumber': '2/2', + }) + assert out['disc_number'] == 2 + + def test_track_number_default_to_one(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + }) + assert out['track_number'] == 1 + assert out['disc_number'] == 1 + + def test_track_number_zero_or_negative_falls_back_to_one(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', 'tracknumber': '0', + }) + assert out['track_number'] == 1 # or-default of 0 → 1 + + def test_track_number_unparseable_falls_back_to_one(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': 'side-a-2', + }) + assert out['track_number'] == 1 + + def test_int_track_number(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': 5, + 'discnumber': 2, + }) + assert out['track_number'] == 5 + assert out['disc_number'] == 2 + + def test_float_track_number_truncated(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': 5.7, + }) + assert out['track_number'] == 5 + + def test_zero_padded_track_number(self): + out = extract_track_meta_from_tags({ + 'title': 't', 'artist': 'a', + 'tracknumber': '03', + }) + assert out['track_number'] == 3 + + def test_non_dict_input(self): + assert extract_track_meta_from_tags(None) is None + assert extract_track_meta_from_tags([]) is None + assert extract_track_meta_from_tags('') is None + + def test_empty_dict(self): + assert extract_track_meta_from_tags({}) is None + + +# ─── extract_album_meta_from_tags ───────────────────────────────────── + + +class TestExtractAlbumMeta: + def test_full_set(self): + out = extract_album_meta_from_tags({ + 'album': 'DAMN.', + 'albumartist': 'Kendrick Lamar', + 'date': '2017-04-14', + 'totaltracks': '14', + 'releasetype': 'Album', + }) + assert out['name'] == 'DAMN.' + assert out['title'] == 'DAMN.' + assert out['album_artist'] == 'Kendrick Lamar' + assert out['release_date'] == '2017' + assert out['total_tracks'] == 14 + assert out['album_type'] == 'album' + assert out['image_url'] == '' + assert out['id'] == '' + + def test_year_normalization_from_full_date(self): + for date_input, expected_year in [ + ('2020-01-15', '2020'), + ('2020', '2020'), + ('2020-01', '2020'), + ('Jan 5, 2020', '2020'), + ('1999/12/31', '1999'), + ]: + out = extract_album_meta_from_tags({'album': 'a', 'date': date_input}) + assert out['release_date'] == expected_year, f"date={date_input!r}" + + def test_year_falls_back_to_year_field(self): + out = extract_album_meta_from_tags({'album': 'a', 'year': '2018'}) + assert out['release_date'] == '2018' + + def test_year_falls_back_to_originaldate(self): + out = extract_album_meta_from_tags({'album': 'a', 'originaldate': '2010'}) + assert out['release_date'] == '2010' + + def test_year_missing_returns_empty(self): + out = extract_album_meta_from_tags({'album': 'a'}) + assert out['release_date'] == '' + + def test_totaltracks_from_id3_shape(self): + # ID3 may store track_number as "5/12" — use the trailing 12. + out = extract_album_meta_from_tags({ + 'album': 'a', 'tracknumber': '5/12', + }) + assert out['total_tracks'] == 12 + + def test_totaltracks_explicit_field_wins(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'totaltracks': '14', 'tracknumber': '5/12', + }) + assert out['total_tracks'] == 14 + + def test_totaltracks_tracktotal_alias(self): + out = extract_album_meta_from_tags({'album': 'a', 'tracktotal': '8'}) + assert out['total_tracks'] == 8 + + def test_releasetype_canonical(self): + for input_val, expected in [ + ('album', 'album'), ('Album', 'album'), + ('single', 'single'), ('Single', 'single'), + ('ep', 'ep'), ('EP', 'ep'), + ('compilation', 'compilation'), + ('soundtrack', ''), # not in canonical set + ('mixtape', ''), + ('', ''), + ]: + out = extract_album_meta_from_tags({ + 'album': 'a', 'releasetype': input_val, + }) + assert out['album_type'] == expected, f"releasetype={input_val!r}" + + def test_total_discs_explicit_field(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'totaldiscs': '2', + }) + assert out['total_discs'] == 2 + + def test_total_discs_from_id3_disc_form(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'discnumber': '1/2', + }) + assert out['total_discs'] == 2 + + def test_total_discs_explicit_wins_over_disc_form(self): + # When both present, take the larger (defensive against drift). + out = extract_album_meta_from_tags({ + 'album': 'a', 'discnumber': '1/2', 'totaldiscs': '3', + }) + assert out['total_discs'] == 3 + + def test_total_discs_missing_zero(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'discnumber': '1', + }) + assert out['total_discs'] == 0 # caller defaults via max() with disc count + + def test_album_artist_underscore_alias(self): + out = extract_album_meta_from_tags({ + 'album': 'a', 'album_artist': 'Foo', + }) + assert out['album_artist'] == 'Foo' + + def test_missing_album_returns_empty_name(self): + out = extract_album_meta_from_tags({}) + assert out['name'] == '' + # All other fields should still be present (zero/empty), so the + # caller's downstream consumer doesn't KeyError. + assert 'release_date' in out + assert 'total_tracks' in out + + def test_non_dict_input_safe(self): + out = extract_album_meta_from_tags(None) # type: ignore + assert out['name'] == '' + + +# ─── read_album_track_from_file ─────────────────────────────────────── + + +class TestReadAlbumTrackFromFile: + def test_unavailable_result_returns_reason(self): + def fake_reader(_p): + return {'available': False, 'reason': 'No file.'} + + a, t, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert a is None and t is None + assert err == 'No file.' + + def test_unavailable_no_reason_falls_back(self): + def fake_reader(_p): + return {'available': False} + + _, _, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert 'Could not read embedded tags' in (err or '') + + def test_non_dict_result_safe(self): + def fake_reader(_p): + return None + + a, t, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert a is None and t is None + assert err + + def test_essentials_missing_track_returns_reason(self): + # Title missing → unmatched. + def fake_reader(_p): + return {'available': True, 'tags': {'artist': 'a', 'album': 'b'}} + + a, t, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert a is None and t is None + assert 'title' in (err or '').lower() + + def test_essentials_missing_album_returns_reason(self): + # Album missing → unmatched even if track meta extracted. + def fake_reader(_p): + return {'available': True, 'tags': {'title': 't', 'artist': 'a'}} + + a, t, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert a is None and t is None + assert 'album' in (err or '').lower() + + def test_full_extraction(self): + def fake_reader(_p): + return { + 'available': True, + 'duration': 234.5, + 'tags': { + 'title': 'HUMBLE.', + 'artist': 'Kendrick Lamar', + 'album': 'DAMN.', + 'albumartist': 'Kendrick Lamar', + 'tracknumber': '4/14', + 'discnumber': '1/1', + 'date': '2017-04-14', + 'releasetype': 'Album', + }, + } + + album, track, err = read_album_track_from_file( + '/fake.flac', read_embedded_tags_fn=fake_reader, + ) + assert err is None + assert track is not None and album is not None + assert track['name'] == 'HUMBLE.' + assert track['track_number'] == 4 + assert track['disc_number'] == 1 + assert track['duration_ms'] == 234500 + assert album['name'] == 'DAMN.' + assert album['release_date'] == '2017' + assert album['total_tracks'] == 14 + assert album['album_type'] == 'album' + + def test_duration_zero_when_missing(self): + def fake_reader(_p): + return { + 'available': True, + 'tags': { + 'title': 't', 'artist': 'a', 'album': 'b', + }, + } + + _, track, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert err is None + assert track['duration_ms'] == 0 + + def test_duration_unparseable_zero(self): + def fake_reader(_p): + return { + 'available': True, + 'duration': 'banana', + 'tags': {'title': 't', 'artist': 'a', 'album': 'b'}, + } + + _, track, err = read_album_track_from_file('fake', read_embedded_tags_fn=fake_reader) + assert err is None + assert track['duration_ms'] == 0 + + def test_empty_path(self): + a, t, err = read_album_track_from_file('') + assert a is None and t is None + assert err + + def test_non_string_path(self): + a, t, err = read_album_track_from_file(None) # type: ignore + assert a is None and t is None + assert err + + +# ─── normalize_resolved_path ────────────────────────────────────────── + + +class TestNormalizeResolvedPath: + def test_returns_path_when_exists(self, tmp_path): + f = tmp_path / 'x.flac' + f.write_bytes(b'') + assert normalize_resolved_path(str(f)) == str(f) + + def test_none_when_missing(self, tmp_path): + assert normalize_resolved_path(str(tmp_path / 'no.flac')) is None + + def test_empty_input_safe(self): + assert normalize_resolved_path('') is None + assert normalize_resolved_path(None) is None + + +# ─── plan_album_reorganize (tag-mode integration) ──────────────────── +# +# Pin the wiring between the planner branch and the tag-source helper +# so the additive-and-optional contract holds: API mode unchanged, +# tag mode produces matched plan items shaped like API mode (so +# downstream post-process treats them identically). + + +def _stub_metadata_service(monkeypatch): + """Inject a minimal `core.metadata_service` so `library_reorganize` + imports cleanly even in the test process where the real metadata + clients aren't wired.""" + if 'core' not in sys.modules: + sys.modules['core'] = types.ModuleType('core') + if 'core.metadata_service' in sys.modules: + return + fake = types.ModuleType('core.metadata_service') + fake.get_album_for_source = lambda *a, **k: {} + fake.get_album_tracks_for_source = lambda *a, **k: [] + fake.get_client_for_source = lambda *a, **k: None + fake.get_primary_source = lambda: 'deezer' + fake.get_source_priority = lambda primary=None: ['deezer', 'spotify', 'itunes'] + sys.modules['core.metadata_service'] = fake + + +class TestPlannerTagModeIntegration: + def test_tag_mode_planner_matches_every_track_with_good_tags(self, monkeypatch): + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + per_path = { + '/a/track1.flac': { + 'available': True, 'duration': 200, + 'tags': { + 'title': 'Song A', 'artist': 'Foo', 'album': 'AlbumX', + 'tracknumber': '1/3', 'discnumber': '1/1', + 'date': '2020', 'releasetype': 'Album', + }, + }, + '/a/track2.flac': { + 'available': True, 'duration': 230, + 'tags': { + 'title': 'Song B', 'artist': 'Foo', 'album': 'AlbumX', + 'tracknumber': '2/3', 'discnumber': '1/1', + 'date': '2020', 'releasetype': 'Album', + }, + }, + } + monkeypatch.setattr( + 'core.library.file_tags.read_embedded_tags', + lambda p: per_path.get(p, {'available': False, 'reason': 'missing'}), + ) + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'Foo', 'title': 'AlbumX'}, + tracks=[ + {'id': 't1', 'title': 'Song A', 'track_number': 1, 'file_path': '/a/track1.flac'}, + {'id': 't2', 'title': 'Song B', 'track_number': 2, 'file_path': '/a/track2.flac'}, + ], + metadata_source='tags', + resolve_file_path_fn=lambda p: p, + ) + assert plan['status'] == 'planned' + assert plan['source'] == 'tags' + assert plan['total_discs'] == 1 + assert len(plan['items']) == 2 + for it in plan['items']: + assert it['matched'] is True + assert it['api_track']['name'] in ('Song A', 'Song B') + assert it['api_album']['name'] == 'AlbumX' + assert it['api_album']['album_type'] == 'album' + + def test_tag_mode_partial_disc_uses_tagged_total_discs(self, monkeypatch): + # User has only disc 2 of a 2-disc album; tags say so. + # max_disc must reflect tagged total so path builder still + # routes into the multi-disc subfolder. + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + monkeypatch.setattr( + 'core.library.file_tags.read_embedded_tags', + lambda p: { + 'available': True, + 'tags': { + 'title': 'Song A', 'artist': 'Foo', 'album': 'Y', + 'tracknumber': '1/8', 'discnumber': '2/2', + 'totaldiscs': '2', + }, + }, + ) + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'Foo', 'title': 'Y'}, + tracks=[{'id': 't1', 'title': 'Song A', 'track_number': 1, 'file_path': '/a.flac'}], + metadata_source='tags', + resolve_file_path_fn=lambda p: p, + ) + assert plan['status'] == 'planned' + assert plan['total_discs'] == 2 + + def test_tag_mode_file_missing_unmatched_with_reason(self, monkeypatch): + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'Foo', 'title': 'X'}, + tracks=[{'id': 't1', 'title': 'Song', 'track_number': 1, 'file_path': '/missing.flac'}], + metadata_source='tags', + resolve_file_path_fn=lambda p: None, # always missing + ) + # All tracks unmatched → no_source_id status, source='tags'. + assert plan['status'] == 'no_source_id' + assert plan['source'] == 'tags' + assert plan['items'][0]['matched'] is False + assert 'no longer exists' in plan['items'][0]['reason'].lower() + + def test_tag_mode_some_match_some_unreadable(self, monkeypatch): + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + per_path = { + '/good.flac': { + 'available': True, + 'tags': {'title': 'Good', 'artist': 'A', 'album': 'X', 'tracknumber': '1/2'}, + }, + '/bad.flac': {'available': False, 'reason': 'unreadable'}, + } + monkeypatch.setattr( + 'core.library.file_tags.read_embedded_tags', + lambda p: per_path.get(p, {'available': False, 'reason': 'missing'}), + ) + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'A', 'title': 'X'}, + tracks=[ + {'id': 'g', 'title': 'Good', 'track_number': 1, 'file_path': '/good.flac'}, + {'id': 'b', 'title': 'Bad', 'track_number': 2, 'file_path': '/bad.flac'}, + ], + metadata_source='tags', + resolve_file_path_fn=lambda p: p, + ) + assert plan['status'] == 'planned' + assert plan['source'] == 'tags' + matched = [it for it in plan['items'] if it['matched']] + unmatched = [it for it in plan['items'] if not it['matched']] + assert len(matched) == 1 + assert len(unmatched) == 1 + assert unmatched[0]['reason'] == 'unreadable' + + def test_tag_mode_without_resolver_returns_no_source_id(self, monkeypatch): + # Defensive: caller forgot to pass resolve_file_path_fn. + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'A', 'title': 'X'}, + tracks=[{'id': 't1', 'title': 'Song', 'track_number': 1, 'file_path': '/a.flac'}], + metadata_source='tags', + resolve_file_path_fn=None, + ) + assert plan['status'] == 'no_source_id' + assert plan['items'][0]['matched'] is False + assert 'requires the file path resolver' in plan['items'][0]['reason'] + + def test_api_mode_unchanged_default(self, monkeypatch): + # Regression guard: omitting metadata_source preserves the API + # path — calls _resolve_source which calls our stubbed + # metadata_service. Should land in 'no_source_id' since stubs + # return empty. + _stub_metadata_service(monkeypatch) + from core import library_reorganize as lr + + plan = lr.plan_album_reorganize( + album_data={'artist_name': 'Foo', 'title': 'Bar'}, + tracks=[{'id': 't1', 'title': 'Song', 'track_number': 1, 'file_path': '/a.flac'}], + ) + # No metadata_source param → defaults to 'api' → empty stubs + # produce no_source_id. + assert plan['status'] == 'no_source_id' + assert plan['source'] is None # never reached the tags branch diff --git a/tests/tools/__init__.py b/tests/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/tools/test_amazon_client.py b/tests/tools/test_amazon_client.py new file mode 100644 index 00000000..a2e21938 --- /dev/null +++ b/tests/tools/test_amazon_client.py @@ -0,0 +1,1024 @@ +"""Unit tests for core/amazon_client.py. + +All network I/O is mocked via a fake session — no real T2Tunes instance needed. + +Run from project root: + python -m pytest tests/tools/test_amazon_client.py -v +""" + +from __future__ import annotations + +import json +import sys +import threading +from pathlib import Path +from typing import Any, Dict, Optional +from unittest.mock import MagicMock, patch + +import pytest + +# Make sure project root is importable when running tests directly. +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from core.amazon_client import ( + Album, + AmazonClient, + AmazonClientError, + Artist, + T2TunesSearchItem, + T2TunesStreamInfo, + Track, + _rate_limit, +) + + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + +TRACK_DOC = { + "asin": "B09XYZ1234", + "title": "Not Like Us", + "artistName": "Kendrick Lamar", + "__type": "track", + "albumName": "GNX", + "albumAsin": "B0ABCDE123", + "duration": 217, + "isrc": "USRC12345678", +} + +ALBUM_DOC = { + "asin": "B0ABCDE123", + "albumAsin": "B0ABCDE123", + "title": "GNX", + "albumName": "GNX", + "artistName": "Kendrick Lamar", + "__type": "album", + "duration": 0, +} + +SEARCH_RESPONSE_TRACKS = { + "results": [ + { + "hits": [ + {"document": TRACK_DOC}, + { + "document": { + "asin": "B09XYZ5678", + "title": "euphoria", + "artistName": "Kendrick Lamar", + "__type": "track", + "albumName": "euphoria", + "albumAsin": "B0ABCDE456", + "duration": 480, + "isrc": "USRC87654321", + } + }, + ] + } + ] +} + +SEARCH_RESPONSE_ALBUMS = { + "results": [{"hits": [{"document": ALBUM_DOC}]}] +} + +SEARCH_RESPONSE_MIXED = { + "results": [ + { + "hits": [ + {"document": TRACK_DOC}, + {"document": ALBUM_DOC}, + ] + } + ] +} + +ALBUM_METADATA_RESPONSE = { + "albumList": [ + { + "asin": "B0ABCDE123", + "title": "GNX", + "image": "https://example.com/cover.jpg", + "trackCount": 12, + "label": "pgLang/Interscope", + "artistName": "Kendrick Lamar", + } + ] +} + +MEDIA_RESPONSE_FLAC = { + "asin": "B09XYZ1234", + "streamable": True, + "decryptionKey": None, + "streamInfo": { + "codec": "FLAC", + "format": "FLAC", + "sampleRate": 44100, + "streamUrl": "https://cdn.example.com/track.flac", + }, + "tags": { + "title": "Not Like Us", + "artist": "Kendrick Lamar", + "album": "GNX", + "isrc": "USRC12345678", + "trackNumber": "3", + "discNumber": "1", + "date": "2024-11-22", + }, +} + +MEDIA_RESPONSE_HIRES = { + "asin": "B09XYZ1234", + "streamable": True, + "decryptionKey": "somekey", + "streamInfo": { + "codec": "FLAC", + "format": "FLAC", + "sampleRate": 96000, + "streamUrl": "https://cdn.example.com/track-hires.flac", + }, + "tags": { + "title": "Not Like Us", + "artist": "Kendrick Lamar", + "album": "GNX", + "isrc": "USRC12345678", + }, +} + +STATUS_UP = {"amazonMusic": "up", "version": "1.0"} +STATUS_DOWN = {"amazonMusic": "down", "version": "1.0"} + + +def _mock_response(data: Any, status_code: int = 200) -> MagicMock: + resp = MagicMock() + resp.status_code = status_code + resp.ok = 200 <= status_code < 400 + resp.json.return_value = data + resp.text = json.dumps(data) + if status_code >= 400: + from requests import HTTPError + exc = HTTPError(response=resp) + resp.raise_for_status.side_effect = exc + else: + resp.raise_for_status.return_value = None + return resp + + +def _make_client(response_map: Optional[Dict[str, Any]] = None) -> AmazonClient: + """Build an AmazonClient with a fake session. + + response_map: path substring → response data (first match wins). + """ + session = MagicMock() + + def _get(url, params=None, timeout=None, **_): + if response_map: + for key, data in response_map.items(): + if key in url: + if isinstance(data, Exception): + raise data + return _mock_response(data) + return _mock_response({"error": "no mock for " + url}, 404) + + session.get.side_effect = _get + with patch("core.amazon_client._rate_limit"): + with patch("core.amazon_client.config_manager") as cfg: + cfg.get.return_value = "" + client = AmazonClient( + base_url="https://test.t2tunes.local", + country="US", + session=session, + ) + client.session = session + return client + + +# --------------------------------------------------------------------------- +# Dataclass construction +# --------------------------------------------------------------------------- + +class TestTrackDataclass: + def test_from_search_hit_basic(self): + t = Track.from_search_hit(TRACK_DOC) + assert t.id == "B09XYZ1234" + assert t.name == "Not Like Us" + assert t.artists == ["Kendrick Lamar"] + assert t.album == "GNX" + assert t.duration_ms == 217_000 + assert t.isrc == "USRC12345678" + assert t.popularity == 0 + + def test_from_search_hit_missing_fields(self): + t = Track.from_search_hit({}) + assert t.id == "" + assert t.name == "" + assert t.artists == ["Unknown Artist"] + assert t.duration_ms == 0 + assert t.isrc is None + + def test_from_stream_info(self): + stream = T2TunesStreamInfo( + asin="B09XYZ1234", + streamable=True, + codec="FLAC", + format="FLAC", + sample_rate=44100, + stream_url="https://cdn.example.com/track.flac", + decryption_key=None, + title="Not Like Us", + artist="Kendrick Lamar", + album="GNX", + isrc="USRC12345678", + ) + t = Track.from_stream_info(stream) + assert t.id == "B09XYZ1234" + assert t.name == "Not Like Us" + assert t.artists == ["Kendrick Lamar"] + assert t.isrc == "USRC12345678" + + def test_from_stream_info_empty_artist(self): + stream = T2TunesStreamInfo( + asin="B1", + streamable=True, + codec="FLAC", + format="FLAC", + sample_rate=44100, + stream_url="https://cdn.example.com/t.flac", + decryption_key=None, + ) + t = Track.from_stream_info(stream) + assert t.artists == ["Unknown Artist"] + + +class TestArtistDataclass: + def test_from_name(self): + a = Artist.from_name("Kendrick Lamar") + assert a.id == "kendrick_lamar" + assert a.name == "Kendrick Lamar" + assert a.genres == [] + assert a.followers == 0 + + def test_from_name_special_chars(self): + a = Artist.from_name("AC/DC") + assert "ac" in a.id + + +class TestAlbumDataclass: + def test_from_search_hit(self): + al = Album.from_search_hit(ALBUM_DOC) + assert al.id == "B0ABCDE123" + assert al.name == "GNX" + assert al.artists == ["Kendrick Lamar"] + assert al.album_type == "album" + + def test_from_search_hit_fallback_asin(self): + al = Album.from_search_hit({"asin": "B0001", "albumName": "Test", "artistName": "X"}) + assert al.id == "B0001" + + def test_from_metadata(self): + meta = ALBUM_METADATA_RESPONSE["albumList"][0] + al = Album.from_metadata(meta, asin="B0ABCDE123") + assert al.id == "B0ABCDE123" + assert al.name == "GNX" + assert al.total_tracks == 12 + assert al.image_url == "https://example.com/cover.jpg" + + +# --------------------------------------------------------------------------- +# T2TunesSearchItem helpers +# --------------------------------------------------------------------------- + +class TestT2TunesSearchItem: + def test_is_track(self): + item = T2TunesSearchItem( + asin="A1", title="T", artist_name="X", item_type="MusicTrack" + ) + assert item.is_track is True + assert item.is_album is False + + def test_is_album(self): + item = T2TunesSearchItem( + asin="A1", title="T", artist_name="X", item_type="MusicAlbum" + ) + assert item.is_album is True + assert item.is_track is False + + def test_ambiguous_type(self): + item = T2TunesSearchItem( + asin="A1", title="T", artist_name="X", item_type="Unknown" + ) + assert item.is_track is False + assert item.is_album is False + + +# --------------------------------------------------------------------------- +# _iter_search_items static method +# --------------------------------------------------------------------------- + +class TestIterSearchItems: + def test_parses_tracks(self): + items = list(AmazonClient._iter_search_items(SEARCH_RESPONSE_TRACKS)) + assert len(items) == 2 + assert items[0].asin == "B09XYZ1234" + assert items[0].title == "Not Like Us" + assert items[0].is_track + + def test_parses_albums(self): + items = list(AmazonClient._iter_search_items(SEARCH_RESPONSE_ALBUMS)) + assert len(items) == 1 + assert items[0].is_album + + def test_skips_missing_asin(self): + resp = {"results": [{"hits": [{"document": {"title": "No ASIN"}}]}]} + items = list(AmazonClient._iter_search_items(resp)) + assert items == [] + + def test_empty_results(self): + items = list(AmazonClient._iter_search_items({"results": []})) + assert items == [] + + def test_wrong_type_raises(self): + with pytest.raises(AmazonClientError): + list(AmazonClient._iter_search_items(["not", "a", "dict"])) + + def test_skips_malformed_hits(self): + resp = { + "results": [ + { + "hits": [ + "not_a_dict", + {"document": None}, + {"document": {"asin": "B1", "__type": "track", "title": "T", "artistName": "A"}}, + ] + } + ] + } + items = list(AmazonClient._iter_search_items(resp)) + assert len(items) == 1 + assert items[0].asin == "B1" + + +# --------------------------------------------------------------------------- +# _parse_stream_info static method +# --------------------------------------------------------------------------- + +class TestParseStreamInfo: + def test_flac_stream(self): + s = AmazonClient._parse_stream_info(MEDIA_RESPONSE_FLAC) + assert s.asin == "B09XYZ1234" + assert s.streamable is True + assert s.codec == "FLAC" + assert s.sample_rate == 44100 + assert s.stream_url == "https://cdn.example.com/track.flac" + assert s.has_decryption_key is False + assert s.title == "Not Like Us" + assert s.isrc == "USRC12345678" + + def test_hires_with_key(self): + s = AmazonClient._parse_stream_info(MEDIA_RESPONSE_HIRES) + assert s.sample_rate == 96000 + assert s.has_decryption_key is True + + def test_typo_stremeable(self): + data = { + "asin": "B1", + "stremeable": True, # typo variant + "streamInfo": {"codec": "OPUS", "format": "OPUS", "streamUrl": "https://x.com/t.opus"}, + "tags": {}, + } + s = AmazonClient._parse_stream_info(data) + assert s.streamable is True + assert s.codec == "OPUS" + + def test_missing_stream_info(self): + s = AmazonClient._parse_stream_info({"asin": "B1"}) + assert s.stream_url == "" + assert s.codec == "" + assert s.sample_rate is None + assert s.has_decryption_key is False + + +# --------------------------------------------------------------------------- +# AmazonClient — HTTP layer +# --------------------------------------------------------------------------- + +class TestStatus: + def test_success(self): + client = _make_client({"/api/status": STATUS_UP}) + with patch("core.amazon_client._rate_limit"): + result = client.status() + assert result["amazonMusic"] == "up" + + def test_http_error_raises(self): + client = _make_client() + client.session.get.side_effect = None + client.session.get.return_value = _mock_response({}, 503) + with pytest.raises(AmazonClientError, match="HTTP 503"): + client.status() + + def test_non_json_raises(self): + resp = MagicMock() + resp.raise_for_status.return_value = None + resp.json.side_effect = ValueError("not json") + resp.text = "error" + client = _make_client() + client.session.get.side_effect = None + client.session.get.return_value = resp + with pytest.raises(AmazonClientError, match="not JSON"): + client.status() + + +class TestIsAuthenticated: + def test_true_when_up(self): + client = _make_client({"/api/status": STATUS_UP}) + assert client.is_authenticated() is True + + def test_false_when_down(self): + client = _make_client({"/api/status": STATUS_DOWN}) + assert client.is_authenticated() is False + + def test_false_on_error(self): + from requests import RequestException + client = _make_client() + client.session.get.side_effect = RequestException("network error") + assert client.is_authenticated() is False + + +class TestReloadConfig: + def test_reloads_fields(self): + with patch("core.amazon_client.config_manager") as cfg: + cfg.get.side_effect = lambda key, default="": { + "amazon.base_url": "https://new.instance.local", + "amazon.country": "GB", + "amazon.preferred_codec": "opus", + }.get(key, default) + client = AmazonClient(session=MagicMock()) + client.reload_config() + assert "new.instance.local" in client.base_url + assert client.country == "GB" + assert client.preferred_codec == "opus" + + +# --------------------------------------------------------------------------- +# AmazonClient — search_raw +# --------------------------------------------------------------------------- + +class TestSearchRaw: + def test_returns_items(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + items = client.search_raw("Kendrick Lamar") + assert len(items) == 2 + + def test_passes_country(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + client.search_raw("test", types="track") + call_kwargs = client.session.get.call_args + assert "country" in str(call_kwargs) + + def test_network_error_raises(self): + from requests import RequestException + client = _make_client() + client.session.get.side_effect = RequestException("timeout") + with pytest.raises(AmazonClientError): + client.search_raw("test") + + +# --------------------------------------------------------------------------- +# AmazonClient — search_tracks / search_artists / search_albums +# --------------------------------------------------------------------------- + +class TestSearchTracks: + def test_returns_track_list(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + tracks = client.search_tracks("Kendrick Lamar") + assert len(tracks) == 2 + assert all(isinstance(t, Track) for t in tracks) + + def test_respects_limit(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + tracks = client.search_tracks("Kendrick Lamar", limit=1) + assert len(tracks) == 1 + + def test_ignores_album_hits(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_ALBUMS}) + with patch("core.amazon_client._rate_limit"): + tracks = client.search_tracks("GNX") + assert tracks == [] + + def test_track_fields(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + tracks = client.search_tracks("Kendrick") + t = tracks[0] + assert t.name == "Not Like Us" + assert t.artists == ["Kendrick Lamar"] + assert t.album == "GNX" + assert t.duration_ms == 217_000 + assert t.isrc == "USRC12345678" + + +class TestSearchArtists: + def test_returns_unique_artists(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Kendrick") + # Both tracks are by Kendrick Lamar — should deduplicate + assert len(artists) == 1 + assert artists[0].name == "Kendrick Lamar" + + def test_returns_artist_dataclass(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Kendrick") + assert isinstance(artists[0], Artist) + + def test_artist_image_from_album(self): + resp = { + "results": [{"hits": [ + {"document": {"asin": "A1", "title": "T1", "artistName": "Kendrick Lamar", + "__type": "track", "albumAsin": "B0ABCDE123"}}, + ]}] + } + client = _make_client({ + "amazon-music/search": resp, + "amazon-music/metadata": ALBUM_METADATA_RESPONSE, + }) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Kendrick") + assert artists[0].image_url == "https://example.com/cover.jpg" + + def test_deduplicates_feat_credits(self): + resp = { + "results": [ + { + "hits": [ + {"document": {"asin": "A1", "title": "T1", "artistName": "Kendrick Lamar", "__type": "track"}}, + {"document": {"asin": "A2", "title": "T2", "artistName": "Kendrick Lamar feat. SZA", "__type": "track"}}, + {"document": {"asin": "A3", "title": "T3", "artistName": "Kendrick Lamar ft. Drake", "__type": "track"}}, + {"document": {"asin": "A4", "title": "T4", "artistName": "SZA featuring Kendrick Lamar", "__type": "track"}}, + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Kendrick") + names = [a.name for a in artists] + assert "Kendrick Lamar" in names + assert "SZA" in names + assert len(artists) == 2 + + def test_respects_limit(self): + resp = { + "results": [ + { + "hits": [ + { + "document": { + "asin": f"B{i}", + "title": f"Song {i}", + "artistName": f"Artist {i}", + "__type": "track", + } + } + for i in range(10) + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + artists = client.search_artists("Various", limit=3) + assert len(artists) == 3 + + +class TestSearchAlbums: + def test_returns_albums(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_ALBUMS}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + assert len(albums) == 1 + assert isinstance(albums[0], Album) + assert albums[0].id == "B0ABCDE123" + + def test_deduplicates_by_asin(self): + resp = { + "results": [ + { + "hits": [ + {"document": {**ALBUM_DOC}}, + {"document": {**ALBUM_DOC}}, # duplicate + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + assert len(albums) == 1 + + def test_ignores_track_hits(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("Kendrick") + assert albums == [] + + def test_strips_explicit_from_album_name(self): + resp = { + "results": [{"hits": [ + {"document": {**ALBUM_DOC, "albumName": "GNX (Explicit)", "title": "GNX (Explicit)"}}, + ]}] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + assert albums[0].name == "GNX" + + def test_keeps_clean_suffix(self): + resp = { + "results": [{"hits": [ + {"document": {**ALBUM_DOC, "albumName": "GNX [Clean]", "title": "GNX [Clean]"}}, + ]}] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + assert albums[0].name == "GNX [Clean]" + + def test_deduplicates_explicit_clean_as_separate(self): + resp = { + "results": [{"hits": [ + {"document": {**ALBUM_DOC, "asin": "B1", "albumAsin": "B1", "albumName": "GNX (Explicit)", "title": "GNX (Explicit)"}}, + {"document": {**ALBUM_DOC, "asin": "B2", "albumAsin": "B2", "albumName": "GNX [Clean]", "title": "GNX [Clean]"}}, + ]}] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.search_albums("GNX") + names = [a.name for a in albums] + assert "GNX" in names # explicit stripped + assert "GNX [Clean]" in names + assert len(albums) == 2 + + +# --------------------------------------------------------------------------- +# AmazonClient — album_metadata / media_from_asin +# --------------------------------------------------------------------------- + +class TestAlbumMetadata: + def test_returns_dict(self): + client = _make_client({"amazon-music/metadata": ALBUM_METADATA_RESPONSE}) + with patch("core.amazon_client._rate_limit"): + meta = client.album_metadata("B0ABCDE123") + assert "albumList" in meta + assert meta["albumList"][0]["title"] == "GNX" + + +class TestMediaFromAsin: + def test_list_response(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + streams = client.media_from_asin("B09XYZ1234") + assert len(streams) == 1 + assert isinstance(streams[0], T2TunesStreamInfo) + assert streams[0].codec == "FLAC" + + def test_single_dict_response(self): + client = _make_client({"amazon-music/media-from-asin": MEDIA_RESPONSE_FLAC}) + with patch("core.amazon_client._rate_limit"): + streams = client.media_from_asin("B09XYZ1234") + assert len(streams) == 1 + + def test_invalid_response_raises(self): + client = _make_client({"amazon-music/media-from-asin": "not a list or dict"}) + with pytest.raises(AmazonClientError, match="Unexpected media"): + client.media_from_asin("B09XYZ1234") + + def test_uses_preferred_codec(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + client.preferred_codec = "opus" + with patch("core.amazon_client._rate_limit"): + client.media_from_asin("B09XYZ1234") + call_kwargs = str(client.session.get.call_args) + assert "opus" in call_kwargs + + def test_codec_override(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + client.media_from_asin("B09XYZ1234", codec="eac3") + call_kwargs = str(client.session.get.call_args) + assert "eac3" in call_kwargs + + +# --------------------------------------------------------------------------- +# AmazonClient — higher-level get_* methods +# --------------------------------------------------------------------------- + +class TestGetTrackDetails: + def _client(self): + return _make_client({ + "amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC], + "amazon-music/metadata": ALBUM_METADATA_RESPONSE, + }) + + def test_returns_spotify_compat_dict(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + details = client.get_track_details("B09XYZ1234") + assert details is not None + assert details["id"] == "B09XYZ1234" + assert details["name"] == "Not Like Us" + assert "artists" in details + assert "album" in details + assert details["is_album_track"] is True + + def test_album_image_populated(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + details = client.get_track_details("B09XYZ1234") + assert details["album"]["images"][0]["url"] == "https://example.com/cover.jpg" + + def test_raw_data_present(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + details = client.get_track_details("B09XYZ1234") + assert "raw_data" in details + assert details["raw_data"]["codec"] == "FLAC" + assert details["raw_data"]["sample_rate"] == 44100 + + def test_returns_none_on_empty_streams(self): + client = _make_client({"amazon-music/media-from-asin": []}) + with patch("core.amazon_client._rate_limit"): + assert client.get_track_details("B09XYZ1234") is None + + def test_returns_none_on_api_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_track_details("B09XYZ1234") is None + + def test_graceful_when_metadata_fails(self): + session = MagicMock() + call_count = {"n": 0} + + def _get(url, params=None, timeout=None, **_): + call_count["n"] += 1 + if "media-from-asin" in url: + return _mock_response([MEDIA_RESPONSE_FLAC]) + return _mock_response({}, 500) + + session.get.side_effect = _get + client = AmazonClient(base_url="https://test.local", session=session) + with patch("core.amazon_client._rate_limit"): + details = client.get_track_details("B09XYZ1234") + assert details is not None + assert details["album"]["images"] == [] + + +class TestGetAlbum: + def _client(self): + return _make_client({"amazon-music/metadata": ALBUM_METADATA_RESPONSE, + "amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + + def test_returns_album_dict(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + album = client.get_album("B0ABCDE123") + assert album is not None + assert album["id"] == "B0ABCDE123" + assert album["name"] == "GNX" + assert album["total_tracks"] == 12 + assert album["label"] == "pgLang/Interscope" + + def test_includes_tracks_by_default(self): + client = self._client() + with patch("core.amazon_client._rate_limit"): + album = client.get_album("B0ABCDE123") + assert "tracks" in album + assert isinstance(album["tracks"], dict) + assert "items" in album["tracks"] + + def test_excludes_tracks_when_flag_false(self): + client = _make_client({"amazon-music/metadata": ALBUM_METADATA_RESPONSE}) + with patch("core.amazon_client._rate_limit"): + album = client.get_album("B0ABCDE123", include_tracks=False) + assert "tracks" not in album + + def test_release_date_backfilled_from_stream_tags(self): + # ALBUM_METADATA_RESPONSE has no release_date — should fall back to track date tag + client = self._client() + with patch("core.amazon_client._rate_limit"): + album = client.get_album("B0ABCDE123") + assert album["release_date"] == "2024-11-22" + + def test_returns_none_on_empty_albumlist(self): + client = _make_client({"amazon-music/metadata": {"albumList": []}}) + with patch("core.amazon_client._rate_limit"): + assert client.get_album("B0ABCDE123") is None + + def test_returns_none_on_api_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_album("B0ABCDE123") is None + + +class TestGetAlbumTracks: + def test_returns_spotify_pagination(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + result = client.get_album_tracks("B09XYZ1234") + assert result is not None + assert "items" in result + assert result["total"] == 1 + assert result["next"] is None + assert result["limit"] == 50 + + def test_item_fields(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + result = client.get_album_tracks("B09XYZ1234") + item = result["items"][0] + assert item["id"] == "B09XYZ1234" + assert item["name"] == "Not Like Us" + assert item["isrc"] == "USRC12345678" + assert item["track_number"] == 3 + assert item["disc_number"] == 1 + + def test_duration_enriched_from_search(self): + search_resp = { + "results": [{"hits": [ + {"document": { + "asin": "B09XYZ1234", "__type": "track", + "title": "Not Like Us", "artistName": "Kendrick Lamar", + "albumAsin": "B09XYZ1234", "duration": 217, + }}, + ]}] + } + client = _make_client({ + "amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC], + "amazon-music/search": search_resp, + }) + with patch("core.amazon_client._rate_limit"): + result = client.get_album_tracks("B09XYZ1234") + assert result["items"][0]["duration_ms"] == 217_000 + + def test_duration_zero_when_search_fails(self): + client = _make_client({"amazon-music/media-from-asin": [MEDIA_RESPONSE_FLAC]}) + with patch("core.amazon_client._rate_limit"): + result = client.get_album_tracks("B09XYZ1234") + assert result["items"][0]["duration_ms"] == 0 + + def test_returns_none_on_api_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_album_tracks("B09XYZ1234") is None + + +class TestGetArtist: + def test_returns_artist_dict(self): + client = _make_client({"amazon-music/search": SEARCH_RESPONSE_TRACKS}) + with patch("core.amazon_client._rate_limit"): + artist = client.get_artist("Kendrick Lamar") + assert artist is not None + assert artist["name"] == "Kendrick Lamar" + assert "genres" in artist + assert "followers" in artist + + def test_exact_match_preferred(self): + resp = { + "results": [ + { + "hits": [ + {"document": {"asin": "A1", "title": "T1", "artistName": "Kendrick Lamar", "__type": "track"}}, + {"document": {"asin": "A2", "title": "T2", "artistName": "Kendrick Lamar Jr.", "__type": "track"}}, + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + artist = client.get_artist("Kendrick Lamar") + assert artist["name"] == "Kendrick Lamar" + + def test_returns_none_when_no_match(self): + client = _make_client({"amazon-music/search": {"results": []}}) + with patch("core.amazon_client._rate_limit"): + assert client.get_artist("Nobody") is None + + def test_returns_none_on_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_artist("Kendrick") is None + + +class TestGetArtistAlbums: + def test_returns_filtered_albums(self): + resp = { + "results": [ + { + "hits": [ + { + "document": { + "asin": "B0ABCDE123", + "albumAsin": "B0ABCDE123", + "title": "GNX", + "albumName": "GNX", + "artistName": "Kendrick Lamar", + "__type": "album", + } + }, + { + "document": { + "asin": "B0ZZZ", + "albumAsin": "B0ZZZ", + "title": "Other Album", + "albumName": "Other Album", + "artistName": "Another Artist", + "__type": "album", + } + }, + ] + } + ] + } + client = _make_client({"amazon-music/search": resp}) + with patch("core.amazon_client._rate_limit"): + albums = client.get_artist_albums("Kendrick Lamar") + assert len(albums) == 1 + assert albums[0].name == "GNX" + + def test_respects_limit(self): + hits = [ + { + "document": { + "asin": f"B{i}", + "albumAsin": f"B{i}", + "albumName": f"Album {i}", + "artistName": "Kendrick Lamar", + "__type": "album", + } + } + for i in range(20) + ] + client = _make_client({"amazon-music/search": {"results": [{"hits": hits}]}}) + with patch("core.amazon_client._rate_limit"): + albums = client.get_artist_albums("Kendrick Lamar", limit=5) + assert len(albums) == 5 + + def test_returns_empty_on_error(self): + client = _make_client() + client.session.get.return_value = _mock_response({}, 500) + with patch("core.amazon_client._rate_limit"): + assert client.get_artist_albums("Kendrick") == [] + + +class TestGetTrackFeatures: + def test_always_none(self): + client = AmazonClient(session=MagicMock()) + assert client.get_track_features("B09XYZ1234") is None + + +# --------------------------------------------------------------------------- +# Rate-limit enforcement +# --------------------------------------------------------------------------- + +class TestRateLimit: + def test_enforces_min_interval(self): + import core.amazon_client as mod + original = mod._last_api_call + sleeps = [] + + def fake_sleep(t): + sleeps.append(t) + + with patch("core.amazon_client.time") as mock_time: + mock_time.monotonic.return_value = mod._last_api_call + 0.1 + mock_time.sleep = fake_sleep + with patch("core.amazon_client.api_call_tracker"): + _rate_limit() + # Should have slept since interval not elapsed + assert len(sleeps) > 0 + mod._last_api_call = original + + def test_no_sleep_when_interval_elapsed(self): + import core.amazon_client as mod + original = mod._last_api_call + sleeps = [] + + with patch("core.amazon_client.time") as mock_time: + mock_time.monotonic.return_value = mod._last_api_call + 10.0 + mock_time.sleep = lambda t: sleeps.append(t) + with patch("core.amazon_client.api_call_tracker"): + _rate_limit() + assert sleeps == [] + mod._last_api_call = original diff --git a/tests/tools/test_amazon_download_client.py b/tests/tools/test_amazon_download_client.py new file mode 100644 index 00000000..f831f268 --- /dev/null +++ b/tests/tools/test_amazon_download_client.py @@ -0,0 +1,847 @@ +"""Unit tests for core/amazon_download_client.py. + +All network I/O and subprocess calls are mocked. +No real T2Tunes instance or ffmpeg binary required. + +Run from project root: + python -m pytest tests/tools/test_amazon_download_client.py -v +""" + +from __future__ import annotations + +import asyncio +import io +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from core.amazon_client import AmazonClientError, T2TunesStreamInfo +from core.amazon_download_client import ( + AmazonDownloadClient, + MIN_AUDIO_BYTES, + _codec_key, + _file_extension, + _quality_label, +) +from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +def run(coro): + return asyncio.run(coro) + + +def _stream_info( + *, + asin: str = "B09XYZ1234", + streamable: bool = True, + codec: str = "FLAC", + sample_rate: int = 44100, + stream_url: str = "https://cdn.example.com/track.enc.flac", + decryption_key: Optional[str] = "deadbeef1234", +) -> T2TunesStreamInfo: + return T2TunesStreamInfo( + asin=asin, + streamable=streamable, + codec=codec, + format=codec, + sample_rate=sample_rate, + stream_url=stream_url, + decryption_key=decryption_key, + title="Not Like Us", + artist="Kendrick Lamar", + album="GNX", + isrc="USRC12345678", + ) + + +def _search_items(n_tracks: int = 2, n_albums: int = 1): + from core.amazon_client import T2TunesSearchItem + items = [ + T2TunesSearchItem( + asin=f"B0TRACK{i}", + title=f"Track {i}", + artist_name="Kendrick Lamar", + item_type="MusicTrack", + album_name="GNX", + album_asin="B0ALBUM1", + duration_seconds=200 + i * 10, + isrc=f"USRC{i:08d}", + ) + for i in range(n_tracks) + ] + items += [ + T2TunesSearchItem( + asin=f"B0ALBUM{j}", + title=f"Album {j}", + artist_name="Kendrick Lamar", + item_type="MusicAlbum", + album_name=f"Album {j}", + album_asin=f"B0ALBUM{j}", + duration_seconds=0, + ) + for j in range(n_albums) + ] + return items + + +def _make_client(tmp_path: Path) -> AmazonDownloadClient: + with patch("core.amazon_download_client.config_manager") as cfg: + cfg.get.return_value = str(tmp_path) + with patch("core.amazon_client.config_manager") as cfg2: + cfg2.get.return_value = "" + client = AmazonDownloadClient(download_path=str(tmp_path)) + return client + + +def _fake_chunked_response(data: bytes, status_code: int = 200) -> MagicMock: + resp = MagicMock() + resp.status_code = status_code + resp.ok = status_code < 400 + resp.headers = {"content-length": str(len(data))} + + chunk_size = 4096 + chunks = [data[i : i + chunk_size] for i in range(0, len(data), chunk_size)] or [b""] + + def iter_content(chunk_size=None): + yield from chunks + + resp.iter_content = iter_content + if status_code >= 400: + from requests import HTTPError + resp.raise_for_status.side_effect = HTTPError(response=resp) + else: + resp.raise_for_status.return_value = None + return resp + + +# --------------------------------------------------------------------------- +# Codec / quality helpers +# --------------------------------------------------------------------------- + +class TestCodecHelpers: + def test_codec_key_lowercases(self): + assert _codec_key("FLAC") == "flac" + assert _codec_key("OGG-Vorbis") == "ogg_vorbis" + assert _codec_key("EAC3") == "eac3" + + def test_file_extension_known_codecs(self): + assert _file_extension("FLAC") == "flac" + assert _file_extension("ogg_vorbis") == "ogg" + assert _file_extension("opus") == "opus" + assert _file_extension("eac3") == "eac3" + assert _file_extension("mp4") == "m4a" + assert _file_extension("aac") == "m4a" + assert _file_extension("mp3") == "mp3" + + def test_file_extension_unknown_falls_back(self): + assert _file_extension("wtf_codec") == "bin" + + def test_quality_label_flac_lossless(self): + assert _quality_label("flac", 44100) == "Lossless" + assert _quality_label("FLAC", 48000) == "Lossless" + + def test_quality_label_flac_hires(self): + assert _quality_label("flac", 96000) == "Hi-Res" + assert _quality_label("flac", 192000) == "Hi-Res" + + def test_quality_label_lossy(self): + assert _quality_label("opus") == "Lossy" + assert _quality_label("eac3") == "Lossy" + assert _quality_label("mp3") == "Lossy" + + +# --------------------------------------------------------------------------- +# is_configured / check_connection +# --------------------------------------------------------------------------- + +class TestIsConfigured: + def test_always_true(self, tmp_path): + client = _make_client(tmp_path) + assert client.is_configured() is True + + +class TestCheckConnection: + def test_true_when_client_authenticated(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.is_authenticated.return_value = True + assert run(client.check_connection()) is True + + def test_false_when_client_down(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.is_authenticated.return_value = False + assert run(client.check_connection()) is False + + def test_false_on_exception(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.is_authenticated.side_effect = Exception("timeout") + assert run(client.check_connection()) is False + + +# --------------------------------------------------------------------------- +# search() +# --------------------------------------------------------------------------- + +class TestSearch: + def test_returns_track_results(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=2, n_albums=0) + client._client.preferred_codec = "flac" + + tracks, albums = run(client.search("Kendrick Lamar")) + + assert len(tracks) == 2 + assert len(albums) == 0 + assert all(isinstance(t, TrackResult) for t in tracks) + + def test_track_fields(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=1, n_albums=0) + client._client.preferred_codec = "flac" + + tracks, _ = run(client.search("Not Like Us")) + t = tracks[0] + + assert t.username == "amazon" + assert "B0TRACK0" in t.filename + assert "||" in t.filename + assert t.artist == "Kendrick Lamar" + assert t.title == "Track 0" + assert t.album == "GNX" + assert t.quality == "Lossless" + assert t.duration == 200_000 + + def test_track_source_metadata(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=1, n_albums=0) + client._client.preferred_codec = "flac" + + tracks, _ = run(client.search("test")) + meta = tracks[0]._source_metadata + + assert meta["asin"] == "B0TRACK0" + assert meta["album_asin"] == "B0ALBUM1" + assert "isrc" in meta + + def test_returns_album_results(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=0, n_albums=2) + client._client.preferred_codec = "flac" + + _, albums = run(client.search("GNX")) + + assert len(albums) == 2 + assert all(isinstance(a, AlbumResult) for a in albums) + + def test_album_deduplication(self, tmp_path): + from core.amazon_client import T2TunesSearchItem + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.preferred_codec = "flac" + # Two hits with same album_asin + dup = T2TunesSearchItem( + asin="B0ALBUM0", + title="GNX", + artist_name="Kendrick Lamar", + item_type="MusicAlbum", + album_asin="B0ALBUM0", + ) + client._client.search_raw.return_value = [dup, dup] + + _, albums = run(client.search("GNX")) + assert len(albums) == 1 + + def test_returns_empty_on_error(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.side_effect = AmazonClientError("fail") + + tracks, albums = run(client.search("anything")) + assert tracks == [] + assert albums == [] + + def test_soulseek_compat_fields(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.search_raw.return_value = _search_items(n_tracks=1, n_albums=0) + client._client.preferred_codec = "flac" + + tracks, _ = run(client.search("test")) + t = tracks[0] + + assert t.free_upload_slots == 999 + assert t.upload_speed == 999_999 + assert t.queue_length == 0 + assert t.size == 0 + + +# --------------------------------------------------------------------------- +# _unique_path +# --------------------------------------------------------------------------- + +class TestUniquePath: + def test_returns_original_when_no_conflict(self, tmp_path): + p = tmp_path / "track.flac" + result = AmazonDownloadClient._unique_path(p) + assert result == p + + def test_appends_counter_on_conflict(self, tmp_path): + p = tmp_path / "track.flac" + p.touch() + result = AmazonDownloadClient._unique_path(p) + assert result != p + assert "(1)" in result.name + + def test_increments_counter(self, tmp_path): + p = tmp_path / "track.flac" + p.touch() + (tmp_path / "track (1).flac").touch() + result = AmazonDownloadClient._unique_path(p) + assert "(2)" in result.name + + +# --------------------------------------------------------------------------- +# _record_to_status +# --------------------------------------------------------------------------- + +class TestRecordToStatus: + def test_fields_mapped(self): + rec = { + "id": "dl-001", + "filename": "B1||Artist - Title", + "state": "downloading", + "progress": 0.5, + "size": 10_000_000, + "transferred": 5_000_000, + "speed": 1_000_000, + "time_remaining": 5, + "file_path": "/tmp/track.flac", + } + status = AmazonDownloadClient._record_to_status(rec) + + assert status.id == "dl-001" + assert status.filename == "B1||Artist - Title" + assert status.username == "amazon" + assert status.state == "downloading" + assert status.progress == 0.5 + assert status.size == 10_000_000 + assert status.transferred == 5_000_000 + assert status.speed == 1_000_000 + assert status.time_remaining == 5 + assert status.file_path == "/tmp/track.flac" + + def test_defaults_for_missing_fields(self): + status = AmazonDownloadClient._record_to_status({}) + assert status.id == "" + assert status.state == "queued" + assert status.progress == 0.0 + assert status.size == 0 + assert status.transferred == 0 + assert status.speed == 0 + assert status.time_remaining is None + assert status.file_path is None + + +# --------------------------------------------------------------------------- +# _stream_to_file +# --------------------------------------------------------------------------- + +class TestStreamToFile: + def test_writes_file_and_returns_size(self, tmp_path): + client = _make_client(tmp_path) + data = b"X" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(data) + + out = tmp_path / "output.flac" + downloaded = client._stream_to_file("https://example.com/t.flac", out, "dl-001") + + assert downloaded == len(data) + assert out.exists() + assert out.read_bytes() == data + + def test_raises_on_http_error(self, tmp_path): + from requests import HTTPError + client = _make_client(tmp_path) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(b"", status_code=403) + + out = tmp_path / "output.flac" + with pytest.raises(HTTPError): + client._stream_to_file("https://example.com/t.flac", out, "dl-001") + + def test_respects_shutdown_check(self, tmp_path): + client = _make_client(tmp_path) + data = b"X" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(data) + client.shutdown_check = lambda: True # trigger immediately + + out = tmp_path / "output.flac" + with pytest.raises(RuntimeError, match="Shutdown"): + client._stream_to_file("https://example.com/t.flac", out, "dl-001") + assert not out.exists() + + def test_updates_engine_progress(self, tmp_path): + import itertools + client = _make_client(tmp_path) + data = b"X" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(data) + engine = MagicMock() + client._engine = engine + + out = tmp_path / "output.flac" + counter = itertools.count(0.0, 1.0) + with patch("core.amazon_download_client.time") as mock_time: + mock_time.monotonic.side_effect = lambda: next(counter) + client._stream_to_file("https://example.com/t.flac", out, "dl-001") + + assert engine.update_record.called + + +# --------------------------------------------------------------------------- +# _decrypt_with_ffmpeg +# --------------------------------------------------------------------------- + +class TestDecryptWithFfmpeg: + def test_calls_ffmpeg_with_key(self, tmp_path): + client = _make_client(tmp_path) + enc = tmp_path / "track.enc.flac" + enc.write_bytes(b"encrypted") + out = tmp_path / "track.flac" + + with patch("shutil.which", return_value="/usr/bin/ffmpeg"): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0, stderr=b"") + client._decrypt_with_ffmpeg(enc, out, "deadbeef1234") + + cmd = mock_run.call_args[0][0] + assert "ffmpeg" in cmd[0] + assert "-decryption_key" in cmd + assert "deadbeef1234" in cmd + assert str(enc) in cmd + assert str(out) in cmd + + def test_raises_on_ffmpeg_failure(self, tmp_path): + client = _make_client(tmp_path) + enc = tmp_path / "track.enc.flac" + enc.write_bytes(b"bad") + out = tmp_path / "track.flac" + + with patch("shutil.which", return_value="/usr/bin/ffmpeg"): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock( + returncode=1, stderr=b"Invalid data found" + ) + with pytest.raises(RuntimeError, match="FFmpeg decryption failed"): + client._decrypt_with_ffmpeg(enc, out, "deadbeef1234") + + def test_raises_when_ffmpeg_missing(self, tmp_path): + client = _make_client(tmp_path) + enc = tmp_path / "track.enc.flac" + out = tmp_path / "track.flac" + + with patch("shutil.which", return_value=None): + # Ensure tools/ffmpeg.exe also absent + with pytest.raises(RuntimeError, match="ffmpeg is required"): + client._decrypt_with_ffmpeg(enc, out, "deadbeef1234") + + def test_uses_tools_ffmpeg_when_not_on_path(self, tmp_path): + client = _make_client(tmp_path) + enc = tmp_path / "track.enc.flac" + enc.write_bytes(b"enc") + out = tmp_path / "track.flac" + + fake_ffmpeg = tmp_path / "ffmpeg.exe" + fake_ffmpeg.touch() + + with patch("shutil.which", return_value=None): + with patch( + "core.amazon_download_client.Path.__file__", + create=True, + ): + import os as _os + is_nt = _os.name == "nt" + ffmpeg_name = "ffmpeg.exe" if is_nt else "ffmpeg" + tools_dir = ROOT / "tools" + tools_ffmpeg = tools_dir / ffmpeg_name + if tools_ffmpeg.exists(): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0, stderr=b"") + client._decrypt_with_ffmpeg(enc, out, "aabbcc") + assert str(tools_ffmpeg) in mock_run.call_args[0][0][0] + + +# --------------------------------------------------------------------------- +# _download_sync — integration of stream + decrypt +# --------------------------------------------------------------------------- + +class TestDownloadSync: + def _setup(self, tmp_path: Path, decryption_key: Optional[str] = "deadbeef"): + client = _make_client(tmp_path) + stream = _stream_info(decryption_key=decryption_key) + client._client = MagicMock() + client._client.media_from_asin.return_value = [stream] + client._client.preferred_codec = "flac" + + audio_data = b"A" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(audio_data) + return client, audio_data + + def test_returns_output_path_on_success(self, tmp_path): + client, audio_data = self._setup(tmp_path) + + with patch("shutil.which", return_value="/usr/bin/ffmpeg"): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0, stderr=b"") + # simulate ffmpeg writing output file + def _ffmpeg_side_effect(cmd, capture_output=False): + # Write dummy decrypted data + out_path = Path(cmd[-1]) + out_path.write_bytes(audio_data) + return MagicMock(returncode=0, stderr=b"") + + mock_run.side_effect = _ffmpeg_side_effect + result = client._download_sync("dl-001", "B09XYZ1234", "Kendrick Lamar - Not Like Us") + + assert result is not None + assert Path(result).exists() + assert Path(result).suffix == ".flac" + + def test_tries_next_codec_when_stream_unavailable(self, tmp_path): + client = _make_client(tmp_path) + # FLAC: no streamable results; Opus: success + flac_stream = _stream_info(streamable=False, codec="FLAC") + opus_stream = _stream_info(streamable=True, codec="OPUS", + stream_url="https://cdn.example.com/t.opus", + decryption_key=None) + client._client = MagicMock() + client._client.preferred_codec = "flac" + + def _media(asin, codec): + if codec == "flac": + return [flac_stream] + if codec == "opus": + return [opus_stream] + return [] + + client._client.media_from_asin.side_effect = _media + + audio_data = b"B" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(audio_data) + + result = client._download_sync("dl-001", "B09XYZ1234", "Kendrick - Track") + + assert result is not None + assert ".opus" in result + + def test_returns_none_when_file_too_small(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.preferred_codec = "flac" + client._client.media_from_asin.return_value = [ + _stream_info(decryption_key=None) + ] + tiny_data = b"X" * 100 # way below MIN_AUDIO_BYTES + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(tiny_data) + + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Title") + assert result is None + + def test_tries_next_codec_when_media_fails(self, tmp_path): + client = _make_client(tmp_path) + client._client = MagicMock() + client._client.preferred_codec = "flac" + + call_count = {"n": 0} + + def _media(asin, codec): + call_count["n"] += 1 + if call_count["n"] == 1: + raise AmazonClientError("quota exceeded") + stream = _stream_info(codec="OPUS", decryption_key=None) + return [stream] + + client._client.media_from_asin.side_effect = _media + + audio_data = b"C" * (MIN_AUDIO_BYTES + 1024) + client.session = MagicMock() + client.session.get.return_value = _fake_chunked_response(audio_data) + + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + assert result is not None + + def test_decryption_failure_tries_next_codec(self, tmp_path): + client, audio_data = self._setup(tmp_path, decryption_key="badkey") + + with patch("shutil.which", return_value="/usr/bin/ffmpeg"): + with patch("subprocess.run") as mock_run: + # First codec (flac) decryption fails; no more codecs succeed + mock_run.return_value = MagicMock(returncode=1, stderr=b"decrypt error") + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + + # All codecs should fail since we return the same bad result for all + assert result is None + + def test_updates_engine_state(self, tmp_path): + client, audio_data = self._setup(tmp_path, decryption_key=None) + engine = MagicMock() + client._engine = engine + + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + + update_calls = engine.update_record.call_args_list + states = [c[0][2].get("state") for c in update_calls if "state" in c[0][2]] + assert "downloading" in states + + def test_final_size_update_syncs_transferred_to_size(self, tmp_path): + """size == transferred after success so monitor bytes-incomplete guard doesn't block.""" + client, audio_data = self._setup(tmp_path, decryption_key=None) + engine = MagicMock() + client._engine = engine + + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + + assert result is not None + # Last update must set size == transferred (both equal the output file size) + final_calls = [ + c[0][2] for c in engine.update_record.call_args_list + if 'size' in c[0][2] and 'transferred' in c[0][2] + and c[0][2]['size'] == c[0][2]['transferred'] + and c[0][2]['size'] > 0 + ] + assert final_calls, "Expected a final engine update with size == transferred > 0" + + def test_clear_stream_skips_ffmpeg(self, tmp_path): + client, audio_data = self._setup(tmp_path, decryption_key=None) + + with patch("subprocess.run") as mock_run: + result = client._download_sync("dl-001", "B09XYZ1234", "Artist - Track") + + mock_run.assert_not_called() + assert result is not None + + def test_safe_filename_sanitisation(self, tmp_path): + client, audio_data = self._setup(tmp_path, decryption_key=None) + result = client._download_sync( + "dl-001", "B09XYZ1234", "Björk / Sigur Rós: Hvarf<>Heim" + ) + assert result is not None + # Path must not contain illegal filesystem chars + path = Path(result) + assert "/" not in path.name + assert "<" not in path.name + assert ">" not in path.name + + +# --------------------------------------------------------------------------- +# download() — async dispatch +# --------------------------------------------------------------------------- + +class TestDownloadDispatch: + def test_raises_without_engine(self, tmp_path): + client = _make_client(tmp_path) + with pytest.raises(RuntimeError, match="_engine"): + run(client.download("amazon", "B09XYZ1234||Artist - Title")) + + def test_returns_none_on_bad_filename(self, tmp_path): + client = _make_client(tmp_path) + client._engine = MagicMock() + result = run(client.download("amazon", "no-pipe-delimiter-here")) + assert result is None + + def test_dispatches_to_engine_worker(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.worker.dispatch.return_value = "dl-abc123" + client._engine = engine + + result = run(client.download("amazon", "B09XYZ1234||Kendrick Lamar - Not Like Us")) + + assert result == "dl-abc123" + dispatch_call = engine.worker.dispatch.call_args + assert dispatch_call[1]["source_name"] == "amazon" + assert dispatch_call[1]["target_id"] == "B09XYZ1234" + assert dispatch_call[1]["display_name"] == "Kendrick Lamar - Not Like Us" + assert dispatch_call[1]["impl_callable"] == client._download_sync + + def test_strips_whitespace_from_asin_and_name(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.worker.dispatch.return_value = "dl-xyz" + client._engine = engine + + run(client.download("amazon", " B09XYZ1234 || Artist - Title ")) + + call = engine.worker.dispatch.call_args + assert call[1]["target_id"] == "B09XYZ1234" + assert call[1]["display_name"] == "Artist - Title" + + def test_set_engine_wires_engine(self, tmp_path): + client = _make_client(tmp_path) + assert client._engine is None + engine = MagicMock() + client.set_engine(engine) + assert client._engine is engine + + def test_set_shutdown_check_wires_callback(self, tmp_path): + client = _make_client(tmp_path) + assert client.shutdown_check is None + check = lambda: False + client.set_shutdown_check(check) + assert client.shutdown_check is check + + def test_set_engine_allows_download_dispatch(self, tmp_path): + """set_engine() must unblock download() — the live failure mode.""" + client = _make_client(tmp_path) + engine = MagicMock() + engine.worker.dispatch.return_value = "dl-wired" + client.set_engine(engine) + result = run(client.download("amazon", "B09XYZ1234||Artist - Title")) + assert result == "dl-wired" + + +# --------------------------------------------------------------------------- +# Status interface +# --------------------------------------------------------------------------- + +class TestStatusInterface: + def test_get_all_downloads_empty_without_engine(self, tmp_path): + client = _make_client(tmp_path) + result = run(client.get_all_downloads()) + assert result == [] + + def test_get_all_downloads_converts_records(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.iter_records_for_source.return_value = iter([{ + "id": "dl-001", + "filename": "B1||A - T", + "state": "complete", + "progress": 1.0, + "size": 5_000_000, + "transferred": 5_000_000, + "speed": 0, + }]) + client._engine = engine + + statuses = run(client.get_all_downloads()) + assert len(statuses) == 1 + assert statuses[0].id == "dl-001" + assert statuses[0].state == "complete" + engine.iter_records_for_source.assert_called_once_with('amazon') + + def test_get_download_status_returns_none_without_engine(self, tmp_path): + client = _make_client(tmp_path) + assert run(client.get_download_status("dl-001")) is None + + def test_get_download_status_hit(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.get_record.return_value = { + "id": "dl-001", + "filename": "B1||A - T", + "state": "downloading", + "progress": 0.7, + "size": 10_000_000, + "transferred": 7_000_000, + "speed": 500_000, + } + client._engine = engine + + status = run(client.get_download_status("dl-001")) + assert status is not None + assert status.id == "dl-001" + assert status.state == "downloading" + assert status.progress == 0.7 + engine.get_record.assert_called_once_with('amazon', 'dl-001') + + def test_get_download_status_miss(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.get_record.return_value = None + client._engine = engine + + assert run(client.get_download_status("nonexistent")) is None + + def test_cancel_returns_false_without_engine(self, tmp_path): + client = _make_client(tmp_path) + assert run(client.cancel_download("dl-001")) is False + + def test_cancel_delegates_to_engine(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.get_record.return_value = {'id': 'dl-001', 'state': 'downloading'} + client._engine = engine + + result = run(client.cancel_download("dl-001", remove=True)) + assert result is True + engine.update_record.assert_called_once_with('amazon', 'dl-001', {'state': 'Cancelled'}) + engine.remove_record.assert_called_once_with('amazon', 'dl-001') + + def test_cancel_returns_false_when_record_not_found(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.get_record.return_value = None + client._engine = engine + + result = run(client.cancel_download("nonexistent")) + assert result is False + engine.update_record.assert_not_called() + + def test_clear_completed_returns_true_without_engine(self, tmp_path): + client = _make_client(tmp_path) + assert run(client.clear_all_completed_downloads()) is True + + def test_clear_completed_removes_terminal_records(self, tmp_path): + client = _make_client(tmp_path) + engine = MagicMock() + engine.iter_records_for_source.return_value = iter([ + {'id': 'dl-001', 'state': 'Completed, Succeeded'}, + {'id': 'dl-002', 'state': 'InProgress, Downloading'}, + {'id': 'dl-003', 'state': 'Errored'}, + ]) + client._engine = engine + + result = run(client.clear_all_completed_downloads()) + assert result is True + # Only terminal records removed + removed_ids = [c[0][1] for c in engine.remove_record.call_args_list] + assert 'dl-001' in removed_ids + assert 'dl-003' in removed_ids + assert 'dl-002' not in removed_ids + + def test_status_methods_no_records(self, tmp_path): + """Engine with no Amazon records returns empty/None gracefully.""" + client = _make_client(tmp_path) + engine = MagicMock() + engine.iter_records_for_source.return_value = iter([]) + engine.get_record.return_value = None + client._engine = engine + + assert run(client.get_all_downloads()) == [] + assert run(client.get_download_status("dl-001")) is None + assert run(client.cancel_download("dl-001")) is False + assert run(client.clear_all_completed_downloads()) is True diff --git a/tests/tools/test_t2tunes_probe.py b/tests/tools/test_t2tunes_probe.py new file mode 100644 index 00000000..9856da32 --- /dev/null +++ b/tests/tools/test_t2tunes_probe.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import Mock + +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) + +from t2tunes_probe import T2TunesClient, T2TunesError # noqa: E402 + + +class _Response: + def __init__(self, payload=None, *, status_code=200, text="", headers=None, url="https://example.test/x"): + self._payload = payload + self.status_code = status_code + self.text = text + self.headers = headers or {} + self.url = url + self.ok = 200 <= status_code < 400 + + def json(self): + if isinstance(self._payload, Exception): + raise self._payload + return self._payload + + def raise_for_status(self): + if not self.ok: + raise RuntimeError(f"{self.status_code} error") + + +class T2TunesProbeTests(unittest.TestCase): + def test_status_uses_api_status_endpoint(self): + session = Mock() + session.get.return_value = _Response({"amazonMusic": "up"}) + client = T2TunesClient("https://t2.example", session=session) + + self.assertTrue(client.amazon_music_is_up()) + session.get.assert_called_once() + self.assertEqual(session.get.call_args.args[0], "https://t2.example/api/status") + + def test_search_flattens_nested_hits(self): + session = Mock() + session.get.return_value = _Response({ + "results": [{ + "hits": [{ + "document": { + "__type": "Track", + "asin": "B001", + "title": "Song", + "artistName": "Artist", + "albumName": "Album", + "duration": 123, + "isrc": "USABC123", + } + }] + }] + }) + client = T2TunesClient("https://t2.example", session=session) + + items = client.search("artist song") + + self.assertEqual(len(items), 1) + self.assertEqual(items[0].asin, "B001") + self.assertTrue(items[0].is_track) + self.assertEqual(items[0].duration_seconds, 123) + self.assertEqual(session.get.call_args.kwargs["params"]["types"], "track,album") + + def test_media_handles_streamable_typo_and_decryption_flag(self): + session = Mock() + session.get.return_value = _Response([{ + "asin": "B001", + "stremeable": True, + "decryptionKey": "abc123", + "tags": { + "title": "Song", + "artist": "Artist", + "album": "Album", + "isrc": "USABC123", + }, + "streamInfo": { + "format": "flac", + "codec": "flac", + "sampleRate": 48000, + "streamUrl": "https://cdn.example/song.flac", + }, + }]) + client = T2TunesClient("https://t2.example", session=session) + + streams = client.media_from_asin("B001") + + self.assertEqual(len(streams), 1) + self.assertTrue(streams[0].streamable) + self.assertTrue(streams[0].has_decryption_key) + self.assertEqual(streams[0].stream_url, "https://cdn.example/song.flac") + + def test_non_json_response_raises_probe_error(self): + session = Mock() + session.get.return_value = _Response(ValueError("not json"), text="Service Unavailable") + client = T2TunesClient("https://t2.example", session=session) + + with self.assertRaises(T2TunesError): + client.status() + + def test_probe_stream_falls_back_to_range_get_when_head_is_blocked(self): + session = Mock() + session.head.return_value = _Response(status_code=405) + session.get.return_value = _Response( + status_code=206, + headers={"content-type": "audio/flac", "content-length": "1"}, + url="https://cdn.example/song.flac", + ) + client = T2TunesClient("https://t2.example", session=session) + + result = client.probe_stream("https://cdn.example/song.flac") + + self.assertTrue(result["ok"]) + self.assertEqual(result["method"], "GET range") + self.assertEqual(result["content_type"], "audio/flac") + self.assertEqual(session.get.call_args.kwargs["headers"], {"Range": "bytes=0-0"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/t2tunes_media_plan.py b/tools/t2tunes_media_plan.py new file mode 100644 index 00000000..531e2f89 --- /dev/null +++ b/tools/t2tunes_media_plan.py @@ -0,0 +1,199 @@ +"""Build a T2Tunes media plan for a track without downloading media. + +This runner calls ``tools.t2tunes_probe.T2TunesClient`` and validates the +parts of a future download source that are safe to exercise live: + +- search resolution +- album metadata lookup +- cover URL discovery + HEAD probe +- per-codec media lookup +- stream URL HEAD/range probe + +It writes a JSON plan that a developer can inspect while building an +integration. It intentionally does not download or decrypt protected media. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +TOOL_DIR = Path(__file__).resolve().parent +if str(TOOL_DIR) not in sys.path: + sys.path.insert(0, str(TOOL_DIR)) + +from t2tunes_probe import T2TunesClient, T2TunesSearchItem # noqa: E402 + + +def _slug(value: str) -> str: + cleaned = "".join(ch.lower() if ch.isalnum() else "-" for ch in value) + parts = [part for part in cleaned.split("-") if part] + return "-".join(parts[:12]) or "t2tunes" + + +def _item_score(item: T2TunesSearchItem, query: str, explicit: Optional[bool]) -> int: + text = f"{item.artist_name} {item.title} {item.album_name}".lower() + score = 0 + for token in query.lower().split(): + if token in text: + score += 10 + if item.is_track: + score += 20 + if explicit is True and "explicit" in text: + score += 50 + if explicit is False and "clean" in text: + score += 50 + if explicit is True and "clean" in text: + score -= 40 + if explicit is False and "explicit" in text: + score -= 40 + return score + + +def _choose_result( + items: Iterable[T2TunesSearchItem], + *, + query: str, + asin: Optional[str], + explicit: Optional[bool], +) -> Optional[T2TunesSearchItem]: + items = list(items) + if asin: + return next((item for item in items if item.asin == asin or item.album_asin == asin), None) + if not items: + return None + return max(items, key=lambda item: _item_score(item, query, explicit)) + + +def _first_album(metadata: Dict[str, Any]) -> Dict[str, Any]: + albums = metadata.get("albumList") + if isinstance(albums, list) and albums: + album = albums[0] + return album if isinstance(album, dict) else {} + return {} + + +def _cover_url_from_metadata(metadata: Dict[str, Any]) -> str: + album = _first_album(metadata) + image = album.get("image") + return image if isinstance(image, str) else "" + + +def _head_url(client: T2TunesClient, url: str) -> Dict[str, Any]: + if not url: + return {"ok": False, "reason": "missing url"} + return client.probe_stream(url) + + +def build_plan( + *, + query: str, + base_url: str, + country: str, + codecs: List[str], + asin: Optional[str], + explicit: Optional[bool], + timeout: int, + probe_urls: bool, +) -> Dict[str, Any]: + client = T2TunesClient(base_url, country=country, timeout=timeout) + status = client.status() + search_items = client.search(query) + selected = _choose_result(search_items, query=query, asin=asin, explicit=explicit) + + plan: Dict[str, Any] = { + "base_url": base_url, + "country": country, + "query": query, + "status": status, + "result_count": len(search_items), + "selected": selected.__dict__ if selected else None, + "album_metadata": None, + "cover": None, + "formats": [], + "notes": [ + "This plan validates API behavior only.", + "It intentionally does not download or decrypt protected media.", + ], + } + + if not selected: + return plan + + album_asin = selected.album_asin or selected.asin + metadata = client.album_metadata(album_asin) + cover_url = _cover_url_from_metadata(metadata) + plan["album_metadata"] = { + "asin": album_asin, + "title": _first_album(metadata).get("title"), + "track_count": _first_album(metadata).get("trackCount"), + "image": cover_url, + "label": _first_album(metadata).get("label"), + } + plan["cover"] = { + "url": cover_url, + "probe": _head_url(client, cover_url) if probe_urls else None, + } + + for codec in codecs: + format_client = T2TunesClient(base_url, country=country, codec=codec, timeout=timeout) + streams = format_client.media_from_asin(selected.asin) + entries = [] + for stream in streams: + entry = stream.__dict__.copy() + entry["stream_probe"] = _head_url(format_client, stream.stream_url) if probe_urls else None + entries.append(entry) + plan["formats"].append({ + "requested_codec": codec, + "stream_count": len(entries), + "streams": entries, + }) + + return plan + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create a safe T2Tunes media plan for a track.") + parser.add_argument("query", help="Search query, e.g. 'Kendrick Lamar Not Like Us'") + parser.add_argument("--base-url", default="https://t2tunes.site") + parser.add_argument("--country", default="US") + parser.add_argument("--codec", action="append", dest="codecs", choices=("flac", "opus", "eac3")) + parser.add_argument("--asin", help="Prefer a specific track or album ASIN from search results") + group = parser.add_mutually_exclusive_group() + group.add_argument("--explicit", action="store_true", help="Prefer explicit search results") + group.add_argument("--clean", action="store_true", help="Prefer clean search results") + parser.add_argument("--timeout", type=int, default=30) + parser.add_argument("--no-probe", action="store_true", help="Do not HEAD-probe cover/stream URLs") + parser.add_argument( + "--output", + help="Optional JSON output path. Defaults to ./testTEST/-plan.json", + ) + args = parser.parse_args() + + explicit = True if args.explicit else False if args.clean else None + plan = build_plan( + query=args.query, + base_url=args.base_url, + country=args.country, + codecs=args.codecs or ["flac", "opus", "eac3"], + asin=args.asin, + explicit=explicit, + timeout=args.timeout, + probe_urls=not args.no_probe, + ) + + text = json.dumps(plan, indent=2, sort_keys=True) + output_path = Path(args.output) if args.output else Path.cwd() / "testTEST" / f"{_slug(args.query)}-plan.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(text + "\n", encoding="utf-8") + plan["_written_to"] = str(output_path) + text = json.dumps(plan, indent=2, sort_keys=True) + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/t2tunes_probe.py b/tools/t2tunes_probe.py new file mode 100644 index 00000000..95e18c09 --- /dev/null +++ b/tools/t2tunes_probe.py @@ -0,0 +1,369 @@ +"""Standalone T2Tunes/TripleTriple API probe. + +This is intentionally not wired into SoulSync's download source registry. +It validates the API surface Tubifarry targets: + + /api/status + /api/amazon-music/search + /api/amazon-music/metadata + /api/amazon-music/media-from-asin + +The probe can inspect returned stream metadata and optionally issue a HEAD +request against the stream URL. It does not download or decrypt audio. +""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Optional +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, urljoin +from urllib.request import Request, urlopen + + +DEFAULT_BASE_URL = "https://t2tunes.site" +DEFAULT_TIMEOUT_SECONDS = 30 + + +class T2TunesError(RuntimeError): + """Raised when the T2Tunes API returns an invalid or failed response.""" + + +class _HttpResponse: + def __init__(self, *, body: bytes, status_code: int, headers: Dict[str, str], url: str) -> None: + self.body = body + self.status_code = status_code + self.headers = headers + self.url = url + self.ok = 200 <= status_code < 400 + self.text = body.decode("utf-8", errors="replace") + + def json(self) -> Any: + return json.loads(self.text) + + def raise_for_status(self) -> None: + if not self.ok: + raise T2TunesError(f"HTTP {self.status_code} for {self.url}") + + +class _UrllibSession: + def get( + self, + url: str, + *, + params: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + timeout: int = DEFAULT_TIMEOUT_SECONDS, + allow_redirects: bool = True, # kept for test/session compatibility + stream: bool = False, # kept for test/session compatibility + ) -> _HttpResponse: + del allow_redirects, stream + if params: + separator = "&" if "?" in url else "?" + url = f"{url}{separator}{urlencode(params)}" + return self._request("GET", url, headers=headers, timeout=timeout) + + def head( + self, + url: str, + *, + headers: Optional[Dict[str, str]] = None, + timeout: int = DEFAULT_TIMEOUT_SECONDS, + allow_redirects: bool = True, # kept for test/session compatibility + ) -> _HttpResponse: + del allow_redirects + return self._request("HEAD", url, headers=headers, timeout=timeout) + + def _request( + self, + method: str, + url: str, + *, + headers: Optional[Dict[str, str]] = None, + timeout: int = DEFAULT_TIMEOUT_SECONDS, + ) -> _HttpResponse: + request = Request(url, headers=headers or {}, method=method) + try: + with urlopen(request, timeout=timeout) as response: + return _HttpResponse( + body=response.read(), + status_code=response.status, + headers={k.lower(): v for k, v in response.headers.items()}, + url=response.url, + ) + except HTTPError as exc: + body = exc.read() if hasattr(exc, "read") else b"" + return _HttpResponse( + body=body, + status_code=exc.code, + headers={k.lower(): v for k, v in exc.headers.items()} if exc.headers else {}, + url=exc.url, + ) + except URLError as exc: + raise T2TunesError(f"Network error for {url}: {exc}") from exc + + +@dataclass(frozen=True) +class T2TunesSearchItem: + asin: str + title: str + artist_name: str + item_type: str + album_name: str = "" + album_asin: str = "" + duration_seconds: int = 0 + isrc: str = "" + + @property + def is_album(self) -> bool: + return "album" in self.item_type.lower() + + @property + def is_track(self) -> bool: + return "track" in self.item_type.lower() + + +@dataclass(frozen=True) +class T2TunesStreamInfo: + asin: str + streamable: bool + codec: str + format: str + sample_rate: Optional[int] + stream_url: str + has_decryption_key: bool + title: str = "" + artist: str = "" + album: str = "" + isrc: str = "" + + +class T2TunesClient: + def __init__( + self, + base_url: str = DEFAULT_BASE_URL, + *, + country: str = "US", + codec: str = "flac", + timeout: int = DEFAULT_TIMEOUT_SECONDS, + session: Optional[Any] = None, + ) -> None: + self.base_url = base_url.rstrip("/") + self.country = country.upper() + self.codec = codec.lower() + self.timeout = timeout + self.session = session or _UrllibSession() + + def status(self) -> Dict[str, Any]: + return self._get_json("/api/status") + + def amazon_music_is_up(self) -> bool: + status = self.status() + return str(status.get("amazonMusic", "")).lower() == "up" + + def search(self, query: str, *, types: str = "track,album") -> List[T2TunesSearchItem]: + data = self._get_json( + "/api/amazon-music/search", + params={ + "query": query, + "types": types, + "country": self.country, + }, + ) + return list(_iter_search_items(data)) + + def album_metadata(self, asin: str) -> Dict[str, Any]: + return self._get_json( + "/api/amazon-music/metadata", + params={ + "asin": asin, + "country": self.country, + }, + ) + + def media_from_asin(self, asin: str) -> List[T2TunesStreamInfo]: + data = self._get_json( + "/api/amazon-music/media-from-asin", + params={ + "asin": asin, + "country": self.country, + "codec": self.codec, + }, + ) + if isinstance(data, list): + return [_media_response_to_stream_info(item) for item in data if isinstance(item, dict)] + if isinstance(data, dict): + return [_media_response_to_stream_info(data)] + raise T2TunesError(f"Unexpected media response type: {type(data).__name__}") + + def probe_stream(self, stream_url: str) -> Dict[str, Any]: + """Probe a stream URL without downloading audio bytes.""" + try: + response = self.session.head(stream_url, timeout=self.timeout, allow_redirects=True) + method = "HEAD" + if response.status_code in (405, 403): + response = self.session.get( + stream_url, + timeout=self.timeout, + allow_redirects=True, + stream=True, + headers={"Range": "bytes=0-0"}, + ) + method = "GET range" + return { + "ok": response.ok, + "method": method, + "status_code": response.status_code, + "content_type": response.headers.get("content-type", ""), + "content_length": response.headers.get("content-length", ""), + "accept_ranges": response.headers.get("accept-ranges", ""), + "final_url": response.url, + } + except Exception as exc: + return { + "ok": False, + "error": str(exc), + } + + def _get_json(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any: + url = urljoin(f"{self.base_url}/", path.lstrip("/")) + headers = { + "Accept": "application/json", + "Referer": self.base_url, + "User-Agent": "SoulSync-T2Tunes-Probe/0.1", + } + try: + response = self.session.get(url, params=params, headers=headers, timeout=self.timeout) + response.raise_for_status() + except Exception as exc: + raise T2TunesError(f"Request failed for {url}: {exc}") from exc + + try: + return response.json() + except ValueError as exc: + preview = response.text[:200].replace("\n", " ") + raise T2TunesError(f"Response was not JSON for {url}: {preview!r}") from exc + + +def _iter_search_items(response: Any) -> Iterable[T2TunesSearchItem]: + if not isinstance(response, dict): + raise T2TunesError(f"Unexpected search response type: {type(response).__name__}") + for result in response.get("results") or []: + if not isinstance(result, dict): + continue + for hit in result.get("hits") or []: + if not isinstance(hit, dict): + continue + document = hit.get("document") + if not isinstance(document, dict): + continue + asin = str(document.get("asin") or "") + if not asin: + continue + yield T2TunesSearchItem( + asin=asin, + title=str(document.get("title") or ""), + artist_name=str(document.get("artistName") or ""), + item_type=str(document.get("__type") or ""), + album_name=str(document.get("albumName") or ""), + album_asin=str(document.get("albumAsin") or ""), + duration_seconds=int(document.get("duration") or 0), + isrc=str(document.get("isrc") or ""), + ) + + +def _media_response_to_stream_info(item: Dict[str, Any]) -> T2TunesStreamInfo: + stream_info = item.get("streamInfo") if isinstance(item.get("streamInfo"), dict) else {} + tags = item.get("tags") if isinstance(item.get("tags"), dict) else {} + return T2TunesStreamInfo( + asin=str(item.get("asin") or ""), + streamable=bool(item.get("streamable") if item.get("streamable") is not None else item.get("stremeable")), + codec=str(stream_info.get("codec") or ""), + format=str(stream_info.get("format") or ""), + sample_rate=stream_info.get("sampleRate") if isinstance(stream_info.get("sampleRate"), int) else None, + stream_url=str(stream_info.get("streamUrl") or ""), + has_decryption_key=bool(item.get("decryptionKey")), + title=str(tags.get("title") or ""), + artist=str(tags.get("artist") or ""), + album=str(tags.get("album") or ""), + isrc=str(tags.get("isrc") or ""), + ) + + +def _print_json(data: Any) -> None: + print(json.dumps(data, indent=2, sort_keys=True, default=lambda o: getattr(o, "__dict__", str(o)))) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Probe a T2Tunes/TripleTriple API instance.") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL) + parser.add_argument("--country", default="US") + parser.add_argument("--codec", default="flac", choices=("flac", "opus", "eac3")) + parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT_SECONDS) + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("status") + + search_parser = sub.add_parser("search") + search_parser.add_argument("query") + search_parser.add_argument("--types", default="track,album") + + metadata_parser = sub.add_parser("metadata") + metadata_parser.add_argument("asin") + + media_parser = sub.add_parser("media") + media_parser.add_argument("asin") + media_parser.add_argument("--probe-stream", action="store_true") + + smoke_parser = sub.add_parser("smoke") + smoke_parser.add_argument("query") + smoke_parser.add_argument("--probe-stream", action="store_true") + + args = parser.parse_args() + client = T2TunesClient(args.base_url, country=args.country, codec=args.codec, timeout=args.timeout) + + if args.command == "status": + _print_json(client.status()) + return 0 + + if args.command == "search": + _print_json([item.__dict__ for item in client.search(args.query, types=args.types)]) + return 0 + + if args.command == "metadata": + _print_json(client.album_metadata(args.asin)) + return 0 + + if args.command == "media": + streams = client.media_from_asin(args.asin) + payload = [stream.__dict__ for stream in streams] + if args.probe_stream: + for index, stream in enumerate(streams): + payload[index]["stream_probe"] = client.probe_stream(stream.stream_url) if stream.stream_url else {"ok": False} + _print_json(payload) + return 0 + + if args.command == "smoke": + status = client.status() + search_items = client.search(args.query) + first = next((item for item in search_items if item.is_track or item.is_album), None) + media = client.media_from_asin(first.asin) if first else [] + payload = { + "status": status, + "amazon_music_up": str(status.get("amazonMusic", "")).lower() == "up", + "result_count": len(search_items), + "first_result": first.__dict__ if first else None, + "media": [stream.__dict__ for stream in media[:3]], + } + if args.probe_stream and media and media[0].stream_url: + payload["first_stream_probe"] = client.probe_stream(media[0].stream_url) + _print_json(payload) + return 0 + + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/web_server.py b/web_server.py index 792a6eb7..3b88ca4c 100644 --- a/web_server.py +++ b/web_server.py @@ -40,7 +40,7 @@ logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, system-info, update check, etc. # Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release. -_SOULSYNC_BASE_VERSION = "2.5.3" +_SOULSYNC_BASE_VERSION = "2.5.4" def _build_version_string(): """Append short commit hash to version when available (e.g. 2.35+abc1234).""" @@ -231,6 +231,7 @@ from core.genius_worker import GeniusWorker from core.tidal_worker import TidalWorker from core.qobuz_worker import QobuzWorker from core.hydrabase_worker import HydrabaseWorker +from core.amazon_worker import AmazonWorker from core.hydrabase_client import HydrabaseClient from core.automation_engine import AutomationEngine @@ -474,10 +475,16 @@ def _add_discover_cache_headers(response): def get_current_profile_id() -> int: - """Get the current profile ID from Flask g context or default to 1""" + """Get the current profile ID from Flask g context or default to 1. + + Background callers (automation engine, sync threads, watchlist + scanner) have no request context, so `g.profile_id` raises + `RuntimeError("Working outside of application context")` rather + than `AttributeError`. Catch both so non-request callers degrade + to the admin profile instead of crashing the handler.""" try: return g.profile_id - except AttributeError: + except (AttributeError, RuntimeError): return 1 @@ -767,6 +774,15 @@ db_update_state = { _db_update_automation_id = None # Set when automation triggers DB update, used by callbacks db_update_lock = threading.Lock() + +def _set_db_update_automation_id(value): + """Setter exposed to extracted automation handlers — keeps the + legacy `_db_update_automation_id` global in sync so the live + DB-update progress callbacks below (which still read the global + directly) emit against the right automation card.""" + global _db_update_automation_id + _db_update_automation_id = value + # Quality Scanner state quality_scanner_state = { "status": "idle", # idle, running, finished, error @@ -906,1501 +922,87 @@ _scan_library_automation_id = None def _register_automation_handlers(): - """Register real SoulSync action handlers with the automation engine.""" + """Register real SoulSync action handlers with the automation engine. + + Per-handler bodies live in ``core.automation.handlers``. This + function wires the dependency-injection surface (clients, + callables, mutable state) into a single ``AutomationDeps`` object + and hands it to ``register_all``. + + NOTE: extraction is in progress. The first batch of handlers + (``process_wishlist`` / ``scan_watchlist`` / ``scan_library``) + has been moved to ``core/automation/handlers/``; the remaining + closures still live below until subsequent commits in the same + branch finish the lift. + """ if not automation_engine: return - def _auto_process_wishlist(config): - try: - _process_wishlist_automatically(automation_id=config.get('_automation_id')) - return {'status': 'completed'} - except Exception as e: - return {'status': 'error', 'error': str(e)} - # Note: wishlist processing is async (batch submitted to executor), stats come via batch completion + from core.automation.deps import AutomationDeps, AutomationState + from core.automation.handlers import register_all as _register_extracted_handlers + + # Mutable shared state previously lived as module-level globals + # (`_scan_library_automation_id`, `_pipeline_running`, etc). + # Keeping the legacy globals AS WELL as the new state object during + # the transitional period so the not-yet-extracted closures still + # work; they'll be removed once the rest of the lift is done. + _automation_state = AutomationState() + + from core.watchlist_scanner import get_watchlist_scanner as _get_watchlist_scanner_fn + + _automation_deps = AutomationDeps( + engine=automation_engine, + state=_automation_state, + config_manager=config_manager, + update_progress=_update_automation_progress, + logger=logger, + get_database=get_database, + spotify_client=spotify_client, + tidal_client=tidal_client, + web_scan_manager=web_scan_manager, + process_wishlist_automatically=_process_wishlist_automatically, + process_watchlist_scan_automatically=_process_watchlist_scan_automatically, + is_wishlist_actually_processing=is_wishlist_actually_processing, + is_watchlist_actually_scanning=is_watchlist_actually_scanning, + get_watchlist_scan_state=lambda: watchlist_scan_state, + run_playlist_discovery_worker=_run_playlist_discovery_worker, + run_sync_task=_run_sync_task, + load_sync_status_file=_load_sync_status_file, + get_deezer_client=_get_deezer_client, + parse_youtube_playlist=parse_youtube_playlist, + get_sync_states=lambda: sync_states, + set_db_update_automation_id=_set_db_update_automation_id, + get_db_update_state=lambda: db_update_state, + db_update_lock=db_update_lock, + db_update_executor=db_update_executor, + run_db_update_task=_run_db_update_task, + run_deep_scan_task=_run_deep_scan_task, + get_duplicate_cleaner_state=lambda: duplicate_cleaner_state, + duplicate_cleaner_lock=duplicate_cleaner_lock, + duplicate_cleaner_executor=duplicate_cleaner_executor, + run_duplicate_cleaner=_run_duplicate_cleaner, + get_quality_scanner_state=lambda: quality_scanner_state, + quality_scanner_lock=quality_scanner_lock, + quality_scanner_executor=quality_scanner_executor, + run_quality_scanner=_run_quality_scanner, + download_orchestrator=download_orchestrator, + run_async=run_async, + tasks_lock=tasks_lock, + get_download_batches=lambda: download_batches, + get_download_tasks=lambda: download_tasks, + sweep_empty_download_directories=_sweep_empty_download_directories, + get_staging_path=get_staging_path, + docker_resolve_path=docker_resolve_path, + get_current_profile_id=get_current_profile_id, + get_watchlist_scanner=_get_watchlist_scanner_fn, + get_app=lambda: app, + get_beatport_data_cache=lambda: beatport_data_cache, + init_automation_progress=_init_automation_progress, + record_progress_history=_auto_progress.record_history, + build_personalized_manager=_build_personalized_manager, + ) + _register_extracted_handlers(_automation_deps) - def _auto_scan_watchlist(config): - try: - pre_state_id = id(watchlist_scan_state) - _process_watchlist_scan_automatically( - automation_id=config.get('_automation_id'), - profile_id=config.get('_profile_id') - ) - # Only report stats if a fresh scan actually ran (state dict was reassigned) - if id(watchlist_scan_state) != pre_state_id: - summary = watchlist_scan_state.get('summary', {}) - return { - 'status': 'completed', - 'artists_scanned': summary.get('total_artists', 0), - 'successful_scans': summary.get('successful_scans', 0), - 'new_tracks_found': summary.get('new_tracks_found', 0), - 'tracks_added_to_wishlist': summary.get('tracks_added_to_wishlist', 0), - } - return {'status': 'completed'} - except Exception as e: - return {'status': 'error', 'error': str(e)} - - def _auto_scan_library(config): - global _scan_library_automation_id - automation_id = config.get('_automation_id') - - if not web_scan_manager: - return {'status': 'error', 'reason': 'Scan manager not available'} - - # If another automation is already tracking the scan, just forward the request - if _scan_library_automation_id is not None: - web_scan_manager.request_scan('Automation trigger (additional batch)') - return {'status': 'skipped', 'reason': 'Scan already being tracked'} - - _scan_library_automation_id = automation_id - - try: - result = web_scan_manager.request_scan('Automation trigger') - scan_status_val = result.get('status', 'unknown') - - if scan_status_val == 'queued': - _update_automation_progress(automation_id, - log_line='Scan already in progress — waiting for completion', log_type='info') - else: - delay = result.get('delay_seconds', 60) - _update_automation_progress(automation_id, - log_line=f'Scan scheduled (debounce: {delay}s)', log_type='info') - - # Unified polling loop — handles debounce → scanning → idle transitions - poll_start = time.time() - scan_started = (scan_status_val == 'queued') # Already scanning if queued - while time.time() - poll_start < 1800: # Max 30 min overall - status = web_scan_manager.get_scan_status() - st = status.get('status') - - if st == 'idle': - break # Scan completed (or finished before we started polling) - - elif st == 'scheduled': - elapsed = int(time.time() - poll_start) - _update_automation_progress(automation_id, - phase=f'Waiting for scan to start... ({elapsed}s)', - progress=min(int(elapsed / 60 * 10), 14)) - time.sleep(2) - - elif st == 'scanning': - if not scan_started: - scan_started = True - _update_automation_progress(automation_id, progress=15, - log_line='Scan triggered on media server', log_type='success') - elapsed = status.get('elapsed_seconds', 0) - max_time = status.get('max_time_seconds', 300) - pct = min(15 + int(elapsed / max_time * 80), 95) - mins, secs = divmod(elapsed, 60) - _update_automation_progress(automation_id, - phase=f'Library scan in progress... ({mins}m {secs}s)', - progress=pct) - time.sleep(5) - - else: - time.sleep(2) # Unknown status, avoid tight loop - else: - # 30-min timeout reached - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Library scan timed out after 30 minutes', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - elapsed = round(time.time() - poll_start, 1) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line='Library scan completed', log_type='success') - - return {'status': 'completed', '_manages_own_progress': True, 'scan_duration_seconds': elapsed} - - except Exception as e: - _update_automation_progress(automation_id, status='error', - phase='Error', log_line=str(e), log_type='error') - return {'status': 'error', 'error': str(e), '_manages_own_progress': True} - - finally: - _scan_library_automation_id = None - - # Self-guards only — prevent duplicate runs of the same operation, - # but allow wishlist processing and watchlist scanning to run concurrently. - # Downloads use bandwidth (Soulseek/Tidal/etc), scans use API calls — different resources. - # The per-call rate limiter handles any API contention during post-processing. - automation_engine.register_action_handler('process_wishlist', _auto_process_wishlist, - guard_fn=lambda: is_wishlist_actually_processing()) - automation_engine.register_action_handler('scan_watchlist', _auto_scan_watchlist, - guard_fn=lambda: is_watchlist_actually_scanning()) - automation_engine.register_action_handler('scan_library', _auto_scan_library, - lambda: _scan_library_automation_id is not None) - - def _auto_refresh_mirrored(config): - """Refresh mirrored playlist(s) from source.""" - db = get_database() - playlist_id = config.get('playlist_id') - refresh_all = config.get('all', False) - auto_id = config.get('_automation_id') - - if refresh_all: - playlists = db.get_mirrored_playlists() - elif playlist_id: - p = db.get_mirrored_playlist(int(playlist_id)) - playlists = [p] if p else [] - else: - return {'status': 'error', 'reason': 'No playlist specified'} - - # Filter out sources that can't be refreshed (no external API) - playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] - - refreshed = 0 - errors = [] - for idx, pl in enumerate(playlists): - try: - source = pl.get('source', '') - source_id = pl.get('source_playlist_id', '') - _update_automation_progress(auto_id, - progress=(idx / max(1, len(playlists))) * 100, - phase=f'Refreshing: "{pl.get("name", "")}"', - current_item=pl.get('name', '')) - tracks = None - - if source == 'spotify': - # Try authenticated API first, fall back to public embed scraper - if spotify_client and spotify_client.is_spotify_authenticated(): - playlist_obj = spotify_client.get_playlist_by_id(source_id) - if playlist_obj and playlist_obj.tracks: - tracks = [] - for t in playlist_obj.tracks: - artist_name = t.artists[0] if t.artists else '' - track_dict = { - 'track_name': t.name or '', - 'artist_name': str(artist_name), - 'album_name': t.album or '', - 'duration_ms': t.duration_ms or 0, - 'source_track_id': t.id or '', - } - # Spotify data IS official — auto-mark as discovered - if t.id: - _album_obj = {'name': t.album or ''} - if getattr(t, 'image_url', None): - _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] - track_dict['extra_data'] = json.dumps({ - 'discovered': True, - 'provider': 'spotify', - 'confidence': 1.0, - 'matched_data': { - 'id': t.id, - 'name': t.name or '', - 'artists': [{'name': str(a)} for a in (t.artists or [])], - 'album': _album_obj, - 'duration_ms': t.duration_ms or 0, - 'image_url': getattr(t, 'image_url', None), - } - }) - tracks.append(track_dict) - - # Fallback: public embed scraper (no auth needed) - if tracks is None: - try: - from core.spotify_public_scraper import scrape_spotify_embed - embed_data = scrape_spotify_embed('playlist', source_id) - if embed_data and not embed_data.get('error') and embed_data.get('tracks'): - embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' - tracks = [] - for t in embed_data['tracks']: - artist_names = [a['name'] for a in t.get('artists', [])] - artist_name = artist_names[0] if artist_names else '' - track_dict = { - 'track_name': t.get('name', ''), - 'artist_name': artist_name, - 'album_name': embed_album, - 'duration_ms': t.get('duration_ms', 0), - 'source_track_id': t.get('id', ''), - } - # Store Spotify track ID hint but don't mark discovered — - # Discover step needs to run for proper album art - if t.get('id'): - track_dict['extra_data'] = json.dumps({ - 'discovered': False, - 'spotify_hint': { - 'id': t['id'], - 'name': t.get('name', ''), - 'artists': t.get('artists', []), - } - }) - tracks.append(track_dict) - except Exception as e: - logger.warning(f"Spotify public scraper fallback failed for {source_id}: {e}") - - elif source == 'spotify_public': - # source_playlist_id is an MD5 hash; extract actual Spotify ID from stored description (URL) - try: - from core.spotify_public_scraper import parse_spotify_url, scrape_spotify_embed - spotify_url = pl.get('description', '') - parsed = parse_spotify_url(spotify_url) if spotify_url else None - - # If Spotify is authenticated, use the full API (auto-discovers with album art) - if parsed and parsed.get('type') == 'playlist' and spotify_client and spotify_client.is_spotify_authenticated(): - playlist_obj = spotify_client.get_playlist_by_id(parsed['id']) - if playlist_obj and playlist_obj.tracks: - tracks = [] - for t in playlist_obj.tracks: - artist_name = t.artists[0] if t.artists else '' - track_dict = { - 'track_name': t.name or '', - 'artist_name': str(artist_name), - 'album_name': t.album or '', - 'duration_ms': t.duration_ms or 0, - 'source_track_id': t.id or '', - } - if t.id: - _album_obj = {'name': t.album or ''} - if getattr(t, 'image_url', None): - _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] - track_dict['extra_data'] = json.dumps({ - 'discovered': True, - 'provider': 'spotify', - 'confidence': 1.0, - 'matched_data': { - 'id': t.id, - 'name': t.name or '', - 'artists': [{'name': str(a)} for a in (t.artists or [])], - 'album': _album_obj, - 'duration_ms': t.duration_ms or 0, - 'image_url': getattr(t, 'image_url', None), - } - }) - tracks.append(track_dict) - - # Fallback: public embed scraper (no auth or album-type URL) - if tracks is None and parsed: - embed_data = scrape_spotify_embed(parsed['type'], parsed['id']) - if embed_data and not embed_data.get('error') and embed_data.get('tracks'): - embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' - tracks = [] - for t in embed_data['tracks']: - artist_names = [a['name'] for a in t.get('artists', [])] - artist_name = artist_names[0] if artist_names else '' - tracks.append({ - 'track_name': t.get('name', ''), - 'artist_name': artist_name, - 'album_name': embed_album, - 'duration_ms': t.get('duration_ms', 0), - 'source_track_id': t.get('id', ''), - }) - # No extra_data — let preservation code keep existing discovery data - except Exception as e: - logger.warning(f"Spotify public playlist refresh failed for {source_id}: {e}") - - elif source == 'deezer': - try: - deezer = _get_deezer_client() - playlist_data = deezer.get_playlist(source_id) - if playlist_data and playlist_data.get('tracks'): - tracks = [] - for t in playlist_data['tracks']: - artist_name = t['artists'][0] if t.get('artists') else '' - tracks.append({ - 'track_name': t.get('name', ''), - 'artist_name': str(artist_name), - 'album_name': t.get('album', ''), - 'duration_ms': t.get('duration_ms', 0), - 'source_track_id': str(t.get('id', '')), - }) - except Exception as e: - logger.warning(f"Deezer playlist refresh failed for {source_id}: {e}") - - elif source == 'tidal': - if not tidal_client or not tidal_client.is_authenticated(): - logger.warning(f"Tidal not authenticated — skipping refresh for '{pl.get('name', '')}'") - _update_automation_progress(auto_id, - log_line=f'Skipped "{pl.get("name", "")}" — Tidal not authenticated', log_type='skip') - continue - full_playlist = tidal_client.get_playlist(source_id) - if full_playlist and full_playlist.tracks: - tracks = [] - for t in full_playlist.tracks: - artist_name = t.artists[0] if t.artists else '' - tracks.append({ - 'track_name': t.name or '', - 'artist_name': str(artist_name), - 'album_name': t.album or '', - 'duration_ms': t.duration_ms or 0, - 'source_track_id': t.id or '', - }) - - elif source == 'youtube': - # source_playlist_id is now a deterministic hash; use stored description (original URL) for refresh - yt_url = pl.get('description', '') or f"https://www.youtube.com/playlist?list={source_id}" - playlist_data = parse_youtube_playlist(yt_url) - if playlist_data and playlist_data.get('tracks'): - tracks = [] - for t in playlist_data['tracks']: - artist_name = t['artists'][0] if t.get('artists') else '' - tracks.append({ - 'track_name': t.get('name', ''), - 'artist_name': str(artist_name), - 'album_name': '', - 'duration_ms': t.get('duration_ms', 0), - 'source_track_id': t.get('id', ''), - }) - - if tracks is not None: - # Compare old vs new track IDs to detect changes - old_tracks = db.get_mirrored_playlist_tracks(pl['id']) if pl.get('id') else [] - old_ids = {t.get('source_track_id') for t in old_tracks if t.get('source_track_id')} - new_ids = {t.get('source_track_id') for t in tracks if t.get('source_track_id')} - - # Preserve existing discovery extra_data for tracks that still exist - old_extra_map = db.get_mirrored_tracks_extra_data_map(pl['id']) if pl.get('id') else {} - for t in tracks: - sid = t.get('source_track_id', '') - if sid and sid in old_extra_map and 'extra_data' not in t: - t['extra_data'] = old_extra_map[sid] - - db.mirror_playlist( - source=source, - source_playlist_id=source_id, - name=pl['name'], - tracks=tracks, - profile_id=pl.get('profile_id', 1), - owner=pl.get('owner'), - image_url=pl.get('image_url'), - ) - refreshed += 1 - - # Emit playlist_changed if tracks actually changed - if old_ids != new_ids: - added_count = len(new_ids - old_ids) - removed_count = len(old_ids - new_ids) - logger.info(f"[AUTOMATION] Playlist changed: '{pl.get('name', '')}' — {added_count} added, {removed_count} removed (old={len(old_ids)}, new={len(new_ids)})") - _update_automation_progress(auto_id, - log_line=f'"{pl.get("name", "")}" — {added_count} added, {removed_count} removed', log_type='success') - try: - if automation_engine: - automation_engine.emit('playlist_changed', { - 'playlist_name': pl.get('name', ''), - 'playlist_id': str(pl.get('id', '')), - 'old_count': str(len(old_ids)), - 'new_count': str(len(new_ids)), - 'added': str(added_count), - 'removed': str(removed_count), - }) - except Exception as e: - logger.debug("playlist_synced automation emit failed: %s", e) - else: - logger.warning(f"[AUTOMATION] No changes: '{pl.get('name', '')}' (tracks={len(old_ids)})") - _update_automation_progress(auto_id, - log_line=f'No changes: "{pl.get("name", "")}"', log_type='skip') - except Exception as e: - errors.append(f"{pl.get('name', '?')}: {str(e)}") - _update_automation_progress(auto_id, - log_line=f'Error: {pl.get("name", "?")} — {str(e)}', log_type='error') - return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors))} - - def _auto_sync_playlist(config): - """Sync a mirrored playlist to media server. - Uses discovered metadata when available, skips undiscovered tracks. - When triggered on a schedule, skips if nothing changed since last sync.""" - auto_id = config.get('_automation_id') - playlist_id = config.get('playlist_id') - if not playlist_id: - return {'status': 'error', 'reason': 'No playlist specified'} - - db = get_database() - pl = db.get_mirrored_playlist(int(playlist_id)) - if not pl: - return {'status': 'error', 'reason': 'Playlist not found'} - - tracks = db.get_mirrored_playlist_tracks(int(playlist_id)) - if not tracks: - return {'status': 'error', 'reason': 'No tracks in playlist'} - - # Count currently discovered tracks for smart-skip check - current_discovered = 0 - for t in tracks: - extra = {} - if t.get('extra_data'): - try: - extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] - except (json.JSONDecodeError, TypeError): - pass - if extra.get('discovered') and extra.get('matched_data'): - current_discovered += 1 - - # Convert mirrored tracks to format expected by _run_sync_task - # Use discovered metadata when available, skip undiscovered tracks - tracks_json = [] - skipped_count = 0 - - for t in tracks: - # Parse extra_data for discovery info - extra = {} - if t.get('extra_data'): - try: - extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] - except (json.JSONDecodeError, TypeError): - pass - - if extra.get('discovered') and extra.get('matched_data'): - # Use official discovered metadata - md = extra['matched_data'] - album_raw = md.get('album', '') - album_obj = album_raw if isinstance(album_raw, dict) else {'name': album_raw or ''} - _track_entry = { - 'name': md.get('name', ''), - 'artists': md.get('artists', [{'name': t.get('artist_name', '')}]), - 'album': album_obj, - 'duration_ms': md.get('duration_ms', 0), - 'id': md.get('id', ''), - } - if md.get('track_number'): - _track_entry['track_number'] = md['track_number'] - if md.get('disc_number'): - _track_entry['disc_number'] = md['disc_number'] - tracks_json.append(_track_entry) - else: - # NOT discovered — try to include using available metadata so the - # track can still be searched on Soulseek and added to wishlist. - # Without this, failed discovery blocks the entire download pipeline. - # - # Priority: spotify_hint (has real Spotify ID from embed scraper) - # > raw playlist fields (only if source_track_id is valid) - hint = extra.get('spotify_hint', {}) - # Build album object with cover art from the mirrored playlist track - track_image = (t.get('image_url') or '').strip() - album_obj = { - 'name': (t.get('album_name') or '').strip(), - 'images': [{'url': track_image, 'height': 300, 'width': 300}] if track_image else [], - } - - if hint.get('id') and hint.get('name'): - # spotify_hint has proper Spotify track ID + metadata from embed scraper - hint_artists = hint.get('artists', []) - if hint_artists and isinstance(hint_artists[0], str): - hint_artists = [{'name': a} for a in hint_artists] - elif hint_artists and isinstance(hint_artists[0], dict): - pass # Already in correct format - else: - hint_artists = [{'name': t.get('artist_name', '')}] - tracks_json.append({ - 'name': hint['name'], - 'artists': hint_artists, - 'album': album_obj, - 'duration_ms': t.get('duration_ms', 0), - 'id': hint['id'], - }) - elif t.get('source_track_id') and (t.get('track_name') or '').strip(): - # Has a valid source ID and track name — usable for wishlist - tracks_json.append({ - 'name': t['track_name'].strip(), - 'artists': [{'name': (t.get('artist_name') or '').strip() or 'Unknown Artist'}], - 'album': album_obj, - 'duration_ms': t.get('duration_ms', 0), - 'id': t['source_track_id'], - }) - else: - skipped_count += 1 # No usable ID or name — truly can't process - - if not tracks_json: - _update_automation_progress(auto_id, - log_line=f'No discovered tracks — {skipped_count} need discovery first', log_type='skip') - return { - 'status': 'skipped', - 'reason': f'No discovered tracks to sync ({skipped_count} tracks need discovery first)', - 'skipped_tracks': str(skipped_count), - } - - # Preflight: hash the track list and compare against last sync - # Skip if the exact same set of tracks was already synced and all matched - import hashlib - track_ids_str = ','.join(sorted(t.get('id', '') for t in tracks_json)) - tracks_hash = hashlib.md5(track_ids_str.encode()).hexdigest() - - sync_id_key = f"auto_mirror_{playlist_id}" - try: - sync_statuses = _load_sync_status_file() - last_status = sync_statuses.get(sync_id_key, {}) - last_hash = last_status.get('tracks_hash', '') - last_matched = last_status.get('matched_tracks', -1) - - if (last_hash == tracks_hash and - last_matched >= len(tracks_json)): - # Exact same tracks, all matched last time — nothing to do - _update_automation_progress(auto_id, - log_line=f'All {len(tracks_json)} tracks unchanged since last sync — skipping', - log_type='skip') - return { - 'status': 'skipped', - 'reason': f'All {len(tracks_json)} tracks unchanged since last sync', - } - except Exception as e: - logger.debug("mirror sync last-status read: %s", e) - - _update_automation_progress(auto_id, progress=50, - phase=f'Syncing "{pl["name"]}"', - log_line=f'{len(tracks_json)} discovered, {skipped_count} skipped', - log_type='info') - - sync_id = f"auto_mirror_{playlist_id}" - _update_automation_progress(auto_id, progress=90, - log_line=f'Starting sync: {len(tracks_json)} tracks', log_type='success') - threading.Thread( - target=_run_sync_task, - args=(sync_id, pl['name'], tracks_json, auto_id, 1, pl.get('image_url', '')), - daemon=True, - name=f'auto-sync-{playlist_id}' - ).start() - return { - 'status': 'started', - 'playlist_name': pl['name'], - 'discovered_tracks': str(len(tracks_json)), - 'skipped_tracks': str(skipped_count), - '_manages_own_progress': True, - } - - def _auto_discover_playlist(config): - """Discover official Spotify/iTunes metadata for mirrored playlist tracks.""" - db = get_database() - playlist_id = config.get('playlist_id') - discover_all = config.get('all', False) - - if discover_all: - playlists = db.get_mirrored_playlists() - elif playlist_id: - p = db.get_mirrored_playlist(int(playlist_id)) - playlists = [p] if p else [] - else: - return {'status': 'error', 'reason': 'No playlist specified'} - - if not playlists: - return {'status': 'error', 'reason': 'No playlists found'} - - threading.Thread( - target=_run_playlist_discovery_worker, - args=(playlists, config.get('_automation_id')), - daemon=True, - name='auto-discover-playlist' - ).start() - names = ', '.join(p['name'] for p in playlists[:3]) - return {'status': 'started', 'playlist_count': str(len(playlists)), 'playlists': names, - '_manages_own_progress': True} - - # --- Playlist Pipeline: single automation for full lifecycle --- - _pipeline_running = False - - def _pipeline_guard(): - return _pipeline_running - - def _auto_playlist_pipeline(config): - """Full playlist lifecycle: refresh → discover → sync → wishlist. - Runs all 4 phases sequentially in one automation, reporting progress throughout.""" - nonlocal _pipeline_running - _pipeline_running = True - automation_id = config.get('_automation_id') - pipeline_start = time.time() - - try: - db = get_database() - playlist_id = config.get('playlist_id') - process_all = config.get('all', False) - skip_wishlist = config.get('skip_wishlist', False) - - # Resolve playlists - if process_all: - playlists = db.get_mirrored_playlists() - elif playlist_id: - p = db.get_mirrored_playlist(int(playlist_id)) - playlists = [p] if p else [] - else: - _pipeline_running = False - return {'status': 'error', 'error': 'No playlist specified'} - - playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] - if not playlists: - _pipeline_running = False - return {'status': 'error', 'error': 'No refreshable playlists found'} - - pl_names = ', '.join(p.get('name', '?') for p in playlists[:3]) - if len(playlists) > 3: - pl_names += f' (+{len(playlists) - 3} more)' - - _update_automation_progress(automation_id, progress=2, - phase=f'Pipeline: {len(playlists)} playlist(s)', - log_line=f'Starting pipeline for: {pl_names}', log_type='info') - - # ── PHASE 1: REFRESH ────────────────────────────────────────── - _update_automation_progress(automation_id, progress=3, - phase='Phase 1/4: Refreshing playlists...', - log_line='Phase 1: Refresh', log_type='info') - - refresh_config = dict(config) - refresh_config['_automation_id'] = None # Don't let sub-handler hijack pipeline progress - refresh_result = _auto_refresh_mirrored(refresh_config) - refreshed = int(refresh_result.get('refreshed', 0)) - refresh_errors = int(refresh_result.get('errors', 0)) - - _update_automation_progress(automation_id, progress=25, - phase='Phase 1/4: Refresh complete', - log_line=f'Phase 1 done: {refreshed} refreshed, {refresh_errors} errors', - log_type='success' if refresh_errors == 0 else 'warning') - - # ── PHASE 2: DISCOVER ───────────────────────────────────────── - _update_automation_progress(automation_id, progress=26, - phase='Phase 2/4: Discovering metadata...', - log_line='Phase 2: Discover', log_type='info') - - # Reload playlists (refresh may have updated them) - if process_all: - disc_playlists = db.get_mirrored_playlists() - else: - disc_playlists = [db.get_mirrored_playlist(int(playlist_id))] - disc_playlists = [p for p in disc_playlists if p] - - # Run discovery in a thread and wait for it - disc_done = threading.Event() - disc_result = {'discovered': 0, 'failed': 0, 'skipped': 0, 'total': 0} - - def _disc_wrapper(pls): - try: - # The worker updates automation_progress internally, - # but we pass None so it doesn't conflict with our pipeline progress - _run_playlist_discovery_worker(pls, automation_id=None) - except Exception as e: - logger.error(f"[Pipeline] Discovery error: {e}") - finally: - disc_done.set() - - threading.Thread(target=_disc_wrapper, args=(disc_playlists,), daemon=True, - name='pipeline-discover').start() - - # Poll for completion with progress updates - poll_start = time.time() - while not disc_done.wait(timeout=3): - elapsed = int(time.time() - poll_start) - _update_automation_progress(automation_id, progress=min(26 + elapsed // 4, 54), - phase=f'Phase 2/4: Discovering... ({elapsed}s)') - if elapsed > 3600: # 1hr safety timeout - _update_automation_progress(automation_id, - log_line='Discovery timed out after 1 hour', log_type='warning') - break - - _update_automation_progress(automation_id, progress=55, - phase='Phase 2/4: Discovery complete', - log_line='Phase 2 done: discovery complete', log_type='success') - - # ── PHASE 3: SYNC ───────────────────────────────────────────── - _update_automation_progress(automation_id, progress=56, - phase='Phase 3/4: Syncing to server...', - log_line='Phase 3: Sync', log_type='info') - - total_synced = 0 - total_skipped = 0 - sync_errors = 0 - - for pl_idx, pl in enumerate(playlists): - pl_id = pl.get('id') - if not pl_id: - continue - - # Build sync config for this playlist (reuse existing sync handler) - sync_config = { - 'playlist_id': str(pl_id), - '_automation_id': None, # Don't let sync handler hijack our progress - } - sync_result = _auto_sync_playlist(sync_config) - sync_status = sync_result.get('status', '') - - if sync_status == 'started': - # Sync launched a background thread — wait for it - sync_id = f"auto_mirror_{pl_id}" - sync_poll_start = time.time() - while time.time() - sync_poll_start < 600: # 10 min per playlist max - if sync_id in sync_states and sync_states[sync_id].get('status') in ('finished', 'complete', 'error', 'failed'): - break - time.sleep(2) - elapsed = int(time.time() - sync_poll_start) - sub_progress = 56 + ((pl_idx + 1) / max(1, len(playlists))) * 29 - _update_automation_progress(automation_id, progress=min(int(sub_progress), 84), - phase=f'Phase 3/4: Syncing "{pl.get("name", "")}" ({elapsed}s)') - - # Check result - ss = sync_states.get(sync_id, {}) - ss_result = ss.get('result', ss.get('progress', {})) - matched = ss_result.get('matched_tracks', 0) if isinstance(ss_result, dict) else 0 - total_synced += int(matched) if matched else 0 - _update_automation_progress(automation_id, - log_line=f'Synced "{pl.get("name", "")}": {matched} tracks matched', - log_type='success') - - elif sync_status == 'skipped': - total_skipped += 1 - reason = sync_result.get('reason', 'unchanged') - _update_automation_progress(automation_id, - log_line=f'Skipped "{pl.get("name", "")}": {reason}', - log_type='skip') - elif sync_status == 'error': - sync_errors += 1 - _update_automation_progress(automation_id, - log_line=f'Sync error "{pl.get("name", "")}": {sync_result.get("reason", "unknown")}', - log_type='error') - - _update_automation_progress(automation_id, progress=85, - phase='Phase 3/4: Sync complete', - log_line=f'Phase 3 done: {total_synced} matched, {total_skipped} skipped, {sync_errors} errors', - log_type='success' if sync_errors == 0 else 'warning') - - # ── PHASE 4: WISHLIST ───────────────────────────────────────── - wishlist_queued = 0 - if not skip_wishlist: - _update_automation_progress(automation_id, progress=86, - phase='Phase 4/4: Processing wishlist...', - log_line='Phase 4: Wishlist', log_type='info') - - try: - if not is_wishlist_actually_processing(): - _process_wishlist_automatically(automation_id=None) - _update_automation_progress(automation_id, - log_line='Wishlist processing triggered', log_type='success') - wishlist_queued = 1 - else: - _update_automation_progress(automation_id, - log_line='Wishlist already running — skipped', log_type='skip') - except Exception as e: - _update_automation_progress(automation_id, - log_line=f'Wishlist error: {e}', log_type='warning') - else: - _update_automation_progress(automation_id, progress=86, - log_line='Phase 4: Wishlist skipped (disabled)', log_type='skip') - - # ── COMPLETE ────────────────────────────────────────────────── - duration = int(time.time() - pipeline_start) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Pipeline complete', - log_line=f'Pipeline finished in {duration // 60}m {duration % 60}s', - log_type='success') - - _pipeline_running = False - return { - 'status': 'completed', - '_manages_own_progress': True, - 'playlists_refreshed': str(refreshed), - 'tracks_discovered': 'completed', - 'tracks_synced': str(total_synced), - 'sync_skipped': str(total_skipped), - 'wishlist_queued': str(wishlist_queued), - 'duration_seconds': str(duration), - } - - except Exception as e: - _pipeline_running = False - _update_automation_progress(automation_id, status='error', progress=100, - phase='Pipeline error', - log_line=f'Pipeline failed: {e}', log_type='error') - return {'status': 'error', 'error': str(e), '_manages_own_progress': True} - - automation_engine.register_action_handler('refresh_mirrored', _auto_refresh_mirrored) - automation_engine.register_action_handler('sync_playlist', _auto_sync_playlist) - automation_engine.register_action_handler('discover_playlist', _auto_discover_playlist) - automation_engine.register_action_handler('playlist_pipeline', _auto_playlist_pipeline, _pipeline_guard) - - # --- Phase 3 action handlers --- - - def _auto_start_database_update(config): - global _db_update_automation_id - automation_id = config.get('_automation_id') - if db_update_state.get('status') == 'running': - return {'status': 'skipped', 'reason': 'Database update already running'} - _db_update_automation_id = automation_id - full = config.get('full_refresh', False) - active_server = config_manager.get_active_media_server() - with db_update_lock: - db_update_state.update({ - "status": "running", "phase": "Initializing...", - "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "" - }) - db_update_executor.submit(_run_db_update_task, full, active_server) - - # Monitor DB update progress (callbacks handle card updates, we just block until done) - time.sleep(1) - poll_start = time.time() - last_progress_time = time.time() - last_progress_val = 0 - while time.time() - poll_start < 7200: # Max 2 hours - time.sleep(3) - with db_update_lock: - status = db_update_state.get('status', 'idle') - current_progress = db_update_state.get('progress', 0) - if status != 'running': - break - # Track stall detection — if no progress change in 10 minutes, warn - if current_progress != last_progress_val: - last_progress_val = current_progress - last_progress_time = time.time() - elif time.time() - last_progress_time > 600: - _update_automation_progress(automation_id, - log_line='Database update appears stalled — waiting...', log_type='warning') - last_progress_time = time.time() # Reset so warning repeats every 10 min - else: - # 2-hour timeout reached - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Database update timed out after 2 hours', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - # Finished/error callback already updated the card — return matching status - with db_update_lock: - final_status = db_update_state.get('status', 'unknown') - if final_status == 'error': - return {'status': 'error', 'reason': db_update_state.get('error_message', 'Unknown error'), '_manages_own_progress': True} - with db_update_lock: - stats = { - 'status': 'completed', 'full_refresh': str(full), '_manages_own_progress': True, - 'artists': db_update_state.get('total', 0), - 'albums': db_update_state.get('total_albums', 0), - 'tracks': db_update_state.get('total_tracks', 0), - 'removed_artists': db_update_state.get('removed_artists', 0), - 'removed_albums': db_update_state.get('removed_albums', 0), - 'removed_tracks': db_update_state.get('removed_tracks', 0), - } - return stats - - def _auto_deep_scan_library(config): - global _db_update_automation_id - automation_id = config.get('_automation_id') - if db_update_state.get('status') == 'running': - return {'status': 'skipped', 'reason': 'Database update already running'} - _db_update_automation_id = automation_id - active_server = config_manager.get_active_media_server() - with db_update_lock: - db_update_state.update({ - "status": "running", "phase": "Deep scan: Initializing...", - "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "" - }) - db_update_executor.submit(_run_deep_scan_task, active_server) - - # Monitor progress (callbacks handle card updates, we just block until done) - time.sleep(1) - poll_start = time.time() - last_progress_time = time.time() - last_progress_val = 0 - while time.time() - poll_start < 7200: # Max 2 hours - time.sleep(3) - with db_update_lock: - status = db_update_state.get('status', 'idle') - current_progress = db_update_state.get('progress', 0) - if status != 'running': - break - if current_progress != last_progress_val: - last_progress_val = current_progress - last_progress_time = time.time() - elif time.time() - last_progress_time > 600: - _update_automation_progress(automation_id, - log_line='Deep scan appears stalled — waiting...', log_type='warning') - last_progress_time = time.time() - else: - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Deep scan timed out after 2 hours', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - with db_update_lock: - final_status = db_update_state.get('status', 'unknown') - if final_status == 'error': - return {'status': 'error', 'reason': db_update_state.get('error_message', 'Unknown error'), '_manages_own_progress': True} - with db_update_lock: - stats = { - 'status': 'completed', '_manages_own_progress': True, - 'artists': db_update_state.get('total', 0), - 'albums': db_update_state.get('total_albums', 0), - 'tracks': db_update_state.get('total_tracks', 0), - 'removed_artists': db_update_state.get('removed_artists', 0), - 'removed_albums': db_update_state.get('removed_albums', 0), - 'removed_tracks': db_update_state.get('removed_tracks', 0), - } - return stats - - def _auto_run_duplicate_cleaner(config): - automation_id = config.get('_automation_id') - if duplicate_cleaner_state.get('status') == 'running': - return {'status': 'skipped', 'reason': 'Duplicate cleaner already running'} - - # Pre-set status before submit so polling loop doesn't see stale 'finished' from last run - with duplicate_cleaner_lock: - duplicate_cleaner_state["status"] = "running" - duplicate_cleaner_executor.submit(_run_duplicate_cleaner) - _update_automation_progress(automation_id, - log_line='Duplicate cleaner started', log_type='info') - - # Monitor duplicate cleaner progress (max 2 hours) - time.sleep(1) # Brief pause for executor to start - poll_start = time.time() - while time.time() - poll_start < 7200: - time.sleep(3) - status = duplicate_cleaner_state.get('status', 'idle') - if status not in ('running',): - break - phase = duplicate_cleaner_state.get('phase', 'Scanning...') - progress = duplicate_cleaner_state.get('progress', 0) - scanned = duplicate_cleaner_state.get('files_scanned', 0) - total = duplicate_cleaner_state.get('total_files', 0) - _update_automation_progress(automation_id, - phase=phase, progress=progress, - processed=scanned, total=total) - else: - # 2-hour timeout reached - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Duplicate cleaner timed out after 2 hours', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - # Check actual exit status (could be 'finished' or 'error') - final_status = duplicate_cleaner_state.get('status', 'idle') - if final_status == 'error': - err = duplicate_cleaner_state.get('error_message', 'Unknown error') - _update_automation_progress(automation_id, status='error', progress=100, - phase='Error', log_line=err, log_type='error') - return {'status': 'error', 'reason': err, '_manages_own_progress': True} - - dupes = duplicate_cleaner_state.get('duplicates_found', 0) - removed = duplicate_cleaner_state.get('deleted', 0) - space_freed = duplicate_cleaner_state.get('space_freed', 0) - scanned = duplicate_cleaner_state.get('files_scanned', 0) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line=f'Found {dupes} duplicates, removed {removed} files', log_type='success') - return { - 'status': 'completed', '_manages_own_progress': True, - 'files_scanned': scanned, - 'duplicates_found': dupes, - 'files_deleted': removed, - 'space_freed_mb': round(space_freed / (1024 * 1024), 1), - } - - def _auto_clear_quarantine(config): - import shutil as _shutil - automation_id = config.get('_automation_id') - quarantine_path = os.path.join(docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), 'ss_quarantine') - if not os.path.exists(quarantine_path): - _update_automation_progress(automation_id, - log_line='No quarantine folder found', log_type='info') - return {'status': 'completed', 'removed': '0'} - removed = 0 - for f in os.listdir(quarantine_path): - fp = os.path.join(quarantine_path, f) - try: - if os.path.isfile(fp): - os.remove(fp) - removed += 1 - elif os.path.isdir(fp): - _shutil.rmtree(fp) - removed += 1 - except Exception as e: - logger.debug("quarantine entry purge failed: %s", e) - _update_automation_progress(automation_id, - log_line=f'Removed {removed} quarantined items', log_type='success' if removed > 0 else 'info') - return {'status': 'completed', 'removed': str(removed)} - - def _auto_cleanup_wishlist(config): - automation_id = config.get('_automation_id') - db = get_database() - removed = db.remove_wishlist_duplicates(get_current_profile_id()) - _update_automation_progress(automation_id, - log_line=f'Removed {removed or 0} duplicate wishlist entries', log_type='success' if removed else 'info') - return {'status': 'completed', 'removed': str(removed or 0)} - - def _auto_update_discovery_pool(config): - automation_id = config.get('_automation_id') - try: - from core.watchlist_scanner import get_watchlist_scanner - scanner = get_watchlist_scanner(spotify_client) - _update_automation_progress(automation_id, - log_line='Updating discovery pool...', log_type='info') - scanner.update_discovery_pool_incremental(get_current_profile_id()) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line='Discovery pool updated', log_type='success') - return {'status': 'completed', '_manages_own_progress': True} - except Exception as e: - _update_automation_progress(automation_id, status='error', - phase='Error', log_line=str(e), log_type='error') - return {'status': 'error', 'reason': str(e), '_manages_own_progress': True} - - def _auto_start_quality_scan(config): - automation_id = config.get('_automation_id') - if quality_scanner_state.get('status') == 'running': - return {'status': 'skipped', 'reason': 'Quality scan already running'} - - scope = config.get('scope', 'watchlist') - # Pre-set status before submit so polling loop doesn't see stale 'finished' from last run - with quality_scanner_lock: - quality_scanner_state["status"] = "running" - quality_scanner_executor.submit(_run_quality_scanner, scope, get_current_profile_id()) - _update_automation_progress(automation_id, - log_line=f'Quality scan started (scope: {scope})', log_type='info') - - # Monitor quality scanner progress (max 2 hours) - time.sleep(1) # Brief pause for executor to start - poll_start = time.time() - while time.time() - poll_start < 7200: - time.sleep(3) - status = quality_scanner_state.get('status', 'idle') - if status not in ('running',): - break - phase = quality_scanner_state.get('phase', 'Scanning...') - progress = quality_scanner_state.get('progress', 0) - processed = quality_scanner_state.get('processed', 0) - total = quality_scanner_state.get('total', 0) - _update_automation_progress(automation_id, - phase=phase, progress=progress, - processed=processed, total=total) - else: - # 2-hour timeout reached - _update_automation_progress(automation_id, status='error', - phase='Timed out', log_line='Quality scan timed out after 2 hours', log_type='error') - return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} - - # Check actual exit status (could be 'finished' or 'error') - final_status = quality_scanner_state.get('status', 'idle') - if final_status == 'error': - err = quality_scanner_state.get('error_message', 'Unknown error') - _update_automation_progress(automation_id, status='error', progress=100, - phase='Error', log_line=err, log_type='error') - return {'status': 'error', 'reason': err, '_manages_own_progress': True} - - issues = quality_scanner_state.get('low_quality', 0) - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line=f'Quality scan complete — {issues} issues found', log_type='success') - return { - 'status': 'completed', 'scope': scope, '_manages_own_progress': True, - 'tracks_scanned': quality_scanner_state.get('processed', 0), - 'quality_met': quality_scanner_state.get('quality_met', 0), - 'low_quality': issues, - 'matched': quality_scanner_state.get('matched', 0), - } - - def _auto_backup_database(config): - import sqlite3, glob as _glob - automation_id = config.get('_automation_id') - db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') - if not os.path.exists(db_path): - return {'status': 'error', 'reason': 'Database file not found'} - max_backups = 5 - timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') - backup_path = f"{db_path}.backup_{timestamp}" - # Use SQLite backup API for safe hot-copy of active database - src = sqlite3.connect(db_path) - dst = sqlite3.connect(backup_path) - src.backup(dst) - dst.close() - src.close() - size_mb = round(os.path.getsize(backup_path) / (1024 * 1024), 1) - # Rolling cleanup — keep only the newest N backups - existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime) - while len(existing) > max_backups: - try: - os.remove(existing.pop(0)) - except Exception as e: - logger.debug("rolling backup cleanup failed: %s", e) - _update_automation_progress(automation_id, - log_line=f'Backup created: {size_mb}MB ({os.path.basename(backup_path)})', log_type='success') - return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)} - - def _auto_refresh_beatport_cache(config): - """Refresh Beatport homepage cache by calling each endpoint internally.""" - automation_id = config.get('_automation_id') - sections = [ - ('hero_tracks', '/api/beatport/hero-tracks', 'Hero Tracks'), - ('new_releases', '/api/beatport/new-releases', 'New Releases'), - ('featured_charts', '/api/beatport/featured-charts', 'Featured Charts'), - ('dj_charts', '/api/beatport/dj-charts', 'DJ Charts'), - ('top_10_lists', '/api/beatport/homepage/top-10-lists', 'Top 10 Lists'), - ('top_10_releases', '/api/beatport/homepage/top-10-releases-cards', 'Top 10 Releases'), - ('hype_picks', '/api/beatport/hype-picks', 'Hype Picks'), - ] - # Invalidate all homepage cache timestamps so endpoints re-scrape - with beatport_data_cache['cache_lock']: - for key in beatport_data_cache['homepage']: - beatport_data_cache['homepage'][key]['timestamp'] = 0 - beatport_data_cache['homepage'][key]['data'] = None - - refreshed = 0 - errors = [] - with app.test_client() as client: - for idx, (_, endpoint, label) in enumerate(sections): - _update_automation_progress(automation_id, - progress=(idx / len(sections)) * 100, - phase=f'Scraping: {label}', - current_item=label) - try: - resp = client.get(endpoint) - if resp.status_code == 200: - refreshed += 1 - _update_automation_progress(automation_id, - log_line=f'{label}: cached', log_type='success') - else: - errors.append(label) - _update_automation_progress(automation_id, - log_line=f'{label}: HTTP {resp.status_code}', log_type='error') - except Exception as e: - errors.append(label) - _update_automation_progress(automation_id, - log_line=f'{label}: {str(e)}', log_type='error') - if idx < len(sections) - 1: - time.sleep(2) - - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line=f'Refreshed {refreshed}/{len(sections)} sections', log_type='success') - return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors)), - '_manages_own_progress': True} - - def _auto_clean_search_history(config): - """Remove old searches from Soulseek.""" - automation_id = config.get('_automation_id') - # Skip if soulseek is not the active download source or in hybrid order - dl_mode = config_manager.get('download_source.mode', 'hybrid') - hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) - soulseek_active = (dl_mode == 'soulseek' or - (dl_mode == 'hybrid' and 'soulseek' in hybrid_order)) - # Reach the underlying SoulseekClient via the orchestrator's - # generic accessor. - slskd = download_orchestrator.client('soulseek') if download_orchestrator else None - if not soulseek_active or not slskd or not slskd.base_url: - _update_automation_progress(automation_id, - log_line='Soulseek not active — skipped', log_type='skip') - return {'status': 'skipped'} - if not config_manager.get('soulseek.auto_clear_searches', True): - _update_automation_progress(automation_id, - log_line='Auto-clear disabled in settings', log_type='skip') - return {'status': 'skipped'} - try: - success = run_async(download_orchestrator.maintain_search_history_with_buffer( - keep_searches=50, trigger_threshold=200 - )) - if success: - _update_automation_progress(automation_id, - log_line='Search history maintenance completed', log_type='success') - return {'status': 'completed'} - else: - _update_automation_progress(automation_id, - log_line='No cleanup needed', log_type='skip') - return {'status': 'completed'} - except Exception as e: - return {'status': 'error', 'error': str(e)} - - def _auto_clean_completed_downloads(config): - """Clear completed downloads and empty directories.""" - automation_id = config.get('_automation_id') - try: - has_active_batches = False - has_post_processing = False - with tasks_lock: - for batch_data in download_batches.values(): - if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: - has_active_batches = True - break - if not has_active_batches: - for task_data in download_tasks.values(): - if task_data.get('status') == 'post_processing': - has_post_processing = True - break - - if has_active_batches: - _update_automation_progress(automation_id, - log_line='Skipped — downloads active', log_type='skip') - return {'status': 'completed'} - - run_async(download_orchestrator.clear_all_completed_downloads()) - if not has_post_processing: - _sweep_empty_download_directories() - _update_automation_progress(automation_id, - log_line='Download cleanup completed', log_type='success') - return {'status': 'completed'} - except Exception as e: - return {'status': 'error', 'reason': str(e)} - - def _auto_full_cleanup(config): - """Run all cleanup tasks: quarantine, download queue, empty dirs, staging, search history.""" - import shutil as _shutil - automation_id = config.get('_automation_id') - steps = [] - - # --- 1. Clear quarantine --- - _update_automation_progress(automation_id, phase='Clearing quarantine...', progress=0) - quarantine_path = os.path.join(docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), 'ss_quarantine') - q_removed = 0 - if os.path.exists(quarantine_path): - for f in os.listdir(quarantine_path): - fp = os.path.join(quarantine_path, f) - try: - if os.path.isfile(fp): - os.remove(fp) - q_removed += 1 - elif os.path.isdir(fp): - _shutil.rmtree(fp) - q_removed += 1 - except Exception as e: - logger.debug("quarantine entry purge failed: %s", e) - steps.append(f'Quarantine: removed {q_removed} items') - _update_automation_progress(automation_id, - log_line=f'Quarantine: removed {q_removed} items', log_type='success' if q_removed else 'info') - - # --- 2. Clear completed/errored/cancelled downloads from Soulseek queue --- - _update_automation_progress(automation_id, phase='Clearing download queue...', progress=20) - has_active_batches = False - has_post_processing = False - with tasks_lock: - for batch_data in download_batches.values(): - if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: - has_active_batches = True - break - if not has_active_batches: - for task_data in download_tasks.values(): - if task_data.get('status') == 'post_processing': - has_post_processing = True - break - if has_active_batches: - steps.append('Download queue: skipped (active batches)') - _update_automation_progress(automation_id, - log_line='Download queue: skipped (active batches)', log_type='skip') - else: - try: - run_async(download_orchestrator.clear_all_completed_downloads()) - steps.append('Download queue: cleared') - _update_automation_progress(automation_id, - log_line='Download queue: cleared', log_type='success') - except Exception as e: - steps.append(f'Download queue: error ({e})') - _update_automation_progress(automation_id, - log_line=f'Download queue: error ({e})', log_type='error') - - # --- 3. Sweep empty download directories --- - _update_automation_progress(automation_id, phase='Sweeping empty directories...', progress=40) - if has_active_batches or has_post_processing: - reason = 'active batches' if has_active_batches else 'post-processing active' - steps.append(f'Empty directories: skipped ({reason})') - _update_automation_progress(automation_id, - log_line=f'Empty directories: skipped ({reason})', log_type='skip') - else: - dirs_removed = _sweep_empty_download_directories() - steps.append(f'Empty directories: removed {dirs_removed}') - _update_automation_progress(automation_id, - log_line=f'Empty directories: removed {dirs_removed}', log_type='success' if dirs_removed else 'info') - - # --- 4. Sweep empty staging directories --- - _update_automation_progress(automation_id, phase='Sweeping import folder...', progress=60) - staging_path = get_staging_path() - s_removed = 0 - if os.path.isdir(staging_path): - for dirpath, _dirnames, _filenames in os.walk(staging_path, topdown=False): - if os.path.normpath(dirpath) == os.path.normpath(staging_path): - continue - try: - entries = os.listdir(dirpath) - except OSError: - continue - visible = [e for e in entries if not e.startswith('.')] - if not visible: - for hidden in entries: - try: - os.remove(os.path.join(dirpath, hidden)) - except Exception as e: - logger.debug("hidden file cleanup failed: %s", e) - try: - os.rmdir(dirpath) - s_removed += 1 - except OSError: - pass - steps.append(f'Staging: removed {s_removed} empty directories') - _update_automation_progress(automation_id, - log_line=f'Staging: removed {s_removed} empty directories', log_type='success' if s_removed else 'info') - - # --- 5. Clean search history (if enabled) --- - _update_automation_progress(automation_id, phase='Cleaning search history...', progress=80) - try: - if not config_manager.get('soulseek.auto_clear_searches', True): - steps.append('Search cleanup: disabled in settings') - _update_automation_progress(automation_id, - log_line='Search cleanup: disabled in settings', log_type='skip') - else: - run_async(download_orchestrator.maintain_search_history_with_buffer( - keep_searches=50, trigger_threshold=200 - )) - steps.append('Search history: cleaned') - _update_automation_progress(automation_id, - log_line='Search history: cleaned', log_type='success') - except Exception as e: - steps.append(f'Search history: error ({e})') - _update_automation_progress(automation_id, - log_line=f'Search history: error ({e})', log_type='error') - - total_removed = q_removed + s_removed - _update_automation_progress(automation_id, status='finished', progress=100, - phase='Complete', - log_line=f'Full cleanup complete — {total_removed} items removed', log_type='success') - return { - 'status': 'completed', - 'quarantine_removed': str(q_removed), - 'staging_removed': str(s_removed), - 'total_removed': str(total_removed), - 'steps': steps, - '_manages_own_progress': True, - } - - def _auto_run_script(config): - """Execute a user script from the scripts directory.""" - import subprocess as _sp - script_name = config.get('script_name', '') - timeout = min(int(config.get('timeout', 60)), 300) - automation_id = config.get('_automation_id') - - if not script_name: - return {'status': 'error', 'error': 'No script selected'} - - scripts_dir = docker_resolve_path(config_manager.get('scripts.path', './scripts')) - if not scripts_dir or not os.path.isdir(scripts_dir): - os.makedirs(scripts_dir, exist_ok=True) - return {'status': 'error', 'error': 'Scripts directory is empty. Add scripts to the scripts/ folder.'} - - script_path = os.path.join(scripts_dir, script_name) - script_path = os.path.realpath(script_path) - - # Security: block path traversal - if not script_path.startswith(os.path.realpath(scripts_dir)): - return {'status': 'error', 'error': 'Script path traversal blocked'} - - if not os.path.isfile(script_path): - return {'status': 'error', 'error': f'Script not found: {script_name}'} - - _update_automation_progress(automation_id, phase=f'Running {script_name}...', progress=10) - - # Build environment with SoulSync context - env = os.environ.copy() - event_data = config.get('_event_data') or {} - env['SOULSYNC_EVENT'] = str(event_data.get('type', '')) - env['SOULSYNC_AUTOMATION'] = config.get('_automation_name', '') - env['SOULSYNC_SCRIPTS_DIR'] = scripts_dir - - try: - # Determine how to run the script - if script_path.endswith('.py'): - cmd = ['python', script_path] - elif script_path.endswith('.sh'): - cmd = ['bash', script_path] - else: - cmd = [script_path] - - result = _sp.run( - cmd, - capture_output=True, text=True, timeout=timeout, - cwd=scripts_dir, env=env - ) - - _update_automation_progress(automation_id, phase='Script completed', progress=100) - - stdout = result.stdout[:2000] if result.stdout else '' - stderr = result.stderr[:1000] if result.stderr else '' - - if result.returncode == 0: - logger.info(f"Script '{script_name}' completed (exit 0)") - else: - logger.warning(f"Script '{script_name}' exited with code {result.returncode}") - - return { - 'status': 'completed' if result.returncode == 0 else 'error', - 'exit_code': str(result.returncode), - 'stdout': stdout, - 'stderr': stderr, - 'script': script_name, - } - except _sp.TimeoutExpired: - _update_automation_progress(automation_id, phase='Script timed out', progress=100) - return {'status': 'error', 'error': f'Script timed out after {timeout}s', 'script': script_name} - except Exception as e: - return {'status': 'error', 'error': str(e), 'script': script_name} - - automation_engine.register_action_handler('run_script', _auto_run_script) - automation_engine.register_action_handler('full_cleanup', _auto_full_cleanup) - - automation_engine.register_action_handler('start_database_update', _auto_start_database_update, - lambda: db_update_state.get('status') == 'running') - automation_engine.register_action_handler('deep_scan_library', _auto_deep_scan_library, - lambda: db_update_state.get('status') == 'running') - automation_engine.register_action_handler('run_duplicate_cleaner', _auto_run_duplicate_cleaner, - lambda: duplicate_cleaner_state.get('status') == 'running') - automation_engine.register_action_handler('clear_quarantine', _auto_clear_quarantine) - automation_engine.register_action_handler('cleanup_wishlist', _auto_cleanup_wishlist) - automation_engine.register_action_handler('update_discovery_pool', _auto_update_discovery_pool) - automation_engine.register_action_handler('start_quality_scan', _auto_start_quality_scan, - lambda: quality_scanner_state.get('status') == 'running') - automation_engine.register_action_handler('backup_database', _auto_backup_database) - automation_engine.register_action_handler('refresh_beatport_cache', _auto_refresh_beatport_cache) - automation_engine.register_action_handler('clean_search_history', _auto_clean_search_history) - automation_engine.register_action_handler('clean_completed_downloads', _auto_clean_completed_downloads) - - def _auto_search_and_download(config): - """Search for a track and download the best match.""" - automation_id = config.get('_automation_id') - query = config.get('query', '').strip() - # Event-triggered: pull query from event data (e.g. webhook_received) - if not query: - event_data = config.get('_event_data', {}) - query = (event_data.get('query', '') or '').strip() - if not query: - if automation_id: - _update_automation_progress(automation_id, - log_line='No search query provided', log_type='error') - return {'status': 'error', 'error': 'No search query provided'} - try: - if automation_id: - _update_automation_progress(automation_id, - phase='Searching', log_line=f'Searching: {query}', log_type='info') - result = run_async(download_orchestrator.search_and_download_best(query)) - if result: - if automation_id: - _update_automation_progress(automation_id, - log_line=f'Download started for: {query}', log_type='success') - return {'status': 'completed', 'query': query, 'download_id': result} - else: - if automation_id: - _update_automation_progress(automation_id, - log_line=f'No match found for: {query}', log_type='warning') - return {'status': 'not_found', 'query': query, 'error': 'No match found'} - except Exception as e: - if automation_id: - _update_automation_progress(automation_id, - log_line=f'Error: {e}', log_type='error') - return {'status': 'error', 'query': query, 'error': str(e)} - - automation_engine.register_action_handler('search_and_download', _auto_search_and_download) - - # Register progress tracking callbacks - def _progress_init(aid, name, action_type): - _init_automation_progress(aid, name, action_type) - - def _progress_finish(aid, result): - result_status = result.get('status', '') - # Skip for handlers that manage their own progress lifecycle - # (they call _update_automation_progress(status='finished') themselves) - if result.get('_manages_own_progress'): - return - status = 'error' if result_status == 'error' else 'finished' - msg = result.get('error', result.get('reason', result_status or 'done')) - _update_automation_progress(aid, status=status, progress=100, - phase='Error' if status == 'error' else 'Complete', - log_line=msg, log_type='error' if status == 'error' else 'success') - - def _record_automation_history(aid, result): - """Capture progress state into run history before cleanup clears it.""" - _auto_progress.record_history(aid, result, get_database()) - - automation_engine.register_progress_callbacks(_progress_init, _progress_finish, _update_automation_progress, _record_automation_history) - - # Register permanent callback: when any scan completes, emit library_scan_completed event - # This replaces the hardcoded scan_completion_callback → trigger_automatic_database_update chain - if web_scan_manager: - def _on_library_scan_completed(): - if automation_engine: - server_type = getattr(web_scan_manager, '_current_server_type', None) or 'unknown' - automation_engine.emit('library_scan_completed', { - 'server_type': server_type, - }) - web_scan_manager.add_scan_completion_callback(_on_library_scan_completed) logger.info("Automation action handlers registered") @@ -3149,21 +1751,22 @@ def _find_downloaded_file(download_path, track_data): audio_extensions = {'.mp3', '.flac', '.ogg', '.aac', '.wma', '.wav', '.m4a'} target_filename = extract_filename(track_data.get('filename', '')) - # YOUTUBE/TIDAL/QOBUZ/HIFI SUPPORT: Handle encoded filename format "id||title" + # YOUTUBE/TIDAL/QOBUZ/HIFI/AMAZON SUPPORT: Handle encoded filename format "id||title" # The file on disk will be "title.ext", not "id||title" is_youtube = track_data.get('username') == 'youtube' is_tidal = track_data.get('username') == 'tidal' is_qobuz = track_data.get('username') == 'qobuz' is_hifi = track_data.get('username') == 'hifi' - is_streaming_source = is_youtube or is_tidal or is_qobuz or is_hifi + is_amazon = track_data.get('username') == 'amazon' + is_streaming_source = is_youtube or is_tidal or is_qobuz or is_hifi or is_amazon target_filename_youtube = None if is_streaming_source and '||' in target_filename: _, title = target_filename.split('||', 1) - if is_tidal or is_qobuz or is_hifi: - # Tidal/Qobuz/HiFi files can be flac or m4a — match any audio extension + if is_tidal or is_qobuz or is_hifi or is_amazon: + # Tidal/Qobuz/HiFi/Amazon files can be flac, opus, eac3, or m4a — match any audio extension safe_title = re.sub(r'[<>:"/\\|?*]', '_', title) target_filename_youtube = safe_title # Extension-less for flexible matching - source_name = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else 'Tidal') + source_name = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else ('Amazon' if is_amazon else 'Tidal')) logger.debug(f"[{source_name} Stream] Looking for file starting with: {target_filename_youtube}") else: # yt-dlp will create "Title.mp3" from "Title" @@ -3201,11 +1804,11 @@ def _find_downloaded_file(download_path, track_data): # For Tidal, compare without extension (file could be .flac or .m4a) compare_target = target_filename_youtube.lower() compare_file = file.lower() - if is_tidal or is_qobuz or is_hifi: + if is_tidal or is_qobuz or is_hifi or is_amazon: compare_file = os.path.splitext(compare_file)[0] similarity = SequenceMatcher(None, compare_file, compare_target).ratio() - source_label = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else ('Tidal' if is_tidal else 'YouTube')) + source_label = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else ('Amazon' if is_amazon else ('Tidal' if is_tidal else 'YouTube'))) logger.debug(f"[{source_label} Stream] Comparing: '{file}' vs '{target_filename_youtube}' = {similarity:.2f}") # Keep track of best match @@ -3225,7 +1828,7 @@ def _find_downloaded_file(download_path, track_data): # For YouTube/Tidal, if we found a good enough match (80%+), use it if is_streaming_source and best_match and best_similarity >= 0.80: - source_label = 'Qobuz' if is_qobuz else ('Tidal' if is_tidal else 'YouTube') + source_label = 'Qobuz' if is_qobuz else ('Amazon' if is_amazon else ('Tidal' if is_tidal else 'YouTube')) logger.debug(f"Found good match ({best_similarity:.2f}) for {source_label} streaming file: {best_match}") return best_match @@ -3256,6 +1859,7 @@ SERVICE_CONFIG_REGISTRY = { 'spotify': {'required': ['client_id', 'client_secret']}, 'itunes': {'always': True}, # default storefront works anon 'deezer': {'always': True}, # anon search works, premium ARL is optional + 'amazon': {'always': True}, # T2Tunes proxy, no credentials required 'discogs': {'required': ['token']}, 'tidal': {'custom': lambda _svc: _tidal_has_auth_token()}, 'qobuz': {'any_of': [['email', 'password'], ['token'], ['user_auth_token']]}, @@ -3401,6 +2005,7 @@ def _get_enrichment_status(): ('genius', 'Genius', lambda: genius_worker), ('audiodb', 'AudioDB', lambda: audiodb_worker), ('discogs', 'Discogs', lambda: discogs_worker), + ('amazon_enrichment', 'Amazon Music', lambda: amazon_worker), ] # Config-based "configured" checks for services that need API keys/credentials @@ -3520,7 +2125,7 @@ def get_status(): # don't depend on slskd being reachable — when one of these is the # active source, surface "connected" without probing slskd so the # dashboard / sidebar indicator stays green. - serverless_sources = ('youtube', 'hifi', 'qobuz', 'tidal', 'deezer_dl', 'lidarr', 'soundcloud') + serverless_sources = ('youtube', 'hifi', 'qobuz', 'tidal', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon') is_serverless = (download_mode in serverless_sources or (download_mode == 'hybrid' and hybrid_order and any(s in serverless_sources for s in hybrid_order))) @@ -4147,7 +2752,7 @@ def handle_settings(): if 'active_media_server' in new_settings: config_manager.set_active_media_server(new_settings['active_media_server']) - for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'qobuz', 'hifi_download', 'deezer_download', 'lidarr_download', 'listenbrainz', 'acoustid', 'lastfm', 'genius', 'import', 'lossy_copy', 'listening_stats', 'ui_appearance', 'youtube', 'content_filter', 'itunes', 'm3u_export', 'musicbrainz', 'deezer', 'audiodb', 'metadata', 'hydrabase', 'security', 'discogs', 'library', 'discover', 'wishlist', 'genre_whitelist', 'post_processing']: + for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'qobuz', 'hifi_download', 'deezer_download', 'amazon_download', 'lidarr_download', 'listenbrainz', 'acoustid', 'lastfm', 'genius', 'import', 'lossy_copy', 'listening_stats', 'ui_appearance', 'youtube', 'content_filter', 'itunes', 'm3u_export', 'musicbrainz', 'deezer', 'audiodb', 'metadata', 'hydrabase', 'security', 'discogs', 'library', 'discover', 'wishlist', 'genre_whitelist', 'post_processing']: if service in new_settings: for key, value in new_settings[service].items(): config_manager.set(f'{service}.{key}', value) @@ -7195,7 +5800,7 @@ def start_download(): if download_id: # Register download for post-processing (simple transfer to /Transfer) context_key = _make_context_key(username, filename) - is_streaming_source = username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud') + is_streaming_source = username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon') with matched_context_lock: matched_downloads_context[context_key] = { 'search_result': { @@ -7577,7 +6182,7 @@ def get_download_status(): all_streaming_downloads = run_async(download_orchestrator.get_all_downloads()) for download in all_streaming_downloads: - if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): + if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): source_label = download.username.title() # Convert DownloadStatus to transfer format that frontend expects streaming_transfer = { @@ -8680,6 +7285,13 @@ def _build_source_only_artist_detail(artist_id, artist_name, source): except Exception as e: logger.debug(f"Discogs client resolution failed: {e}") + az = None + try: + from core.metadata.registry import get_amazon_client + az = get_amazon_client() + except Exception as e: + logger.debug(f"Amazon client resolution failed: {e}") + try: lastfm_api_key = config_manager.get('lastfm.api_key', '') or None except Exception: @@ -8693,6 +7305,7 @@ def _build_source_only_artist_detail(artist_id, artist_name, source): deezer_client=dz, itunes_client=it, discogs_client=dc, + amazon_client=az, lastfm_api_key=lastfm_api_key, ) return jsonify(payload), status @@ -8761,17 +7374,17 @@ def get_artist_detail(artist_id): logger.info(f"Found artist: {artist_info['name']} with {len(owned_releases['albums'])} albums") - # Fix artist image URL - logger.info(f"Artist image before fix: '{artist_info.get('image_url')}'") + # Fix artist image URL. + # NOTE: don't log image_url or the full artist_info dict here. + # The fixed URL embeds the media-server token (and the proxy + # variant URL-encodes it), so logging at INFO writes the token + # straight into app.log. Issue: tokens leaked to disk on every + # artist-page render until this was scrubbed. if artist_info.get('image_url'): artist_info['image_url'] = fix_artist_image_url(artist_info['image_url']) - logger.info(f"Artist image after fix: '{artist_info['image_url']}'") else: logger.warning(f"No artist image URL found for {artist_info['name']}") - # Debug final artist data being sent - logger.info(f"Final artist data being sent: {artist_info}") - # Fix image URLs for all albums for album in owned_releases['albums']: if album.get('image_url'): @@ -8798,6 +7411,7 @@ def get_artist_detail(artist_id): 'itunes': artist_info.get('itunes_artist_id'), 'discogs': artist_info.get('discogs_id'), 'hydrabase': artist_info.get('soul_id'), + 'amazon': artist_info.get('amazon_id'), } artist_detail_discography = _get_artist_detail_discography( @@ -11095,12 +9709,20 @@ def reorganize_album_preview(album_id): the apply endpoint, so the preview is guaranteed to match what apply would actually produce. - Optional body param ``source``: when provided, only that metadata - source is queried (no fallback chain).""" + Optional body params: + source: when provided, only that metadata source is queried + (no fallback chain). + mode: 'api' (default — query metadata source) or 'tags' (read + embedded file tags as the source of truth, issue #592).""" try: from core.library_reorganize import preview_album_reorganize data = request.get_json() or {} chosen_source = data.get('source') or None + metadata_source = data.get('mode') or config_manager.get( + 'library.reorganize_metadata_source', 'api' + ) or 'api' + if metadata_source not in ('api', 'tags'): + metadata_source = 'api' transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) result = preview_album_reorganize( album_id=album_id, @@ -11110,6 +9732,7 @@ def reorganize_album_preview(album_id): build_final_path_fn=_build_final_path_for_track, primary_source=chosen_source, strict_source=bool(chosen_source), + metadata_source=metadata_source, ) if result.get('status') == 'no_album': return jsonify({"success": False, "error": "Album not found"}), 404 @@ -11132,11 +9755,21 @@ def reorganize_album_files(album_id): source (optional): per-album source pick (Spotify / iTunes / Deezer / Discogs / Hydrabase). When omitted, the orchestrator uses the configured primary with fallback. + mode (optional): 'api' (default — query metadata source) or + 'tags' (read embedded file tags as the source of truth, + issue #592). When omitted, falls back to the + ``library.reorganize_metadata_source`` config setting, + then to 'api'. """ try: from core.reorganize_queue import get_queue data = request.get_json() or {} chosen_source = data.get('source') or None + metadata_source = data.get('mode') or config_manager.get( + 'library.reorganize_metadata_source', 'api' + ) or 'api' + if metadata_source not in ('api', 'tags'): + metadata_source = 'api' # Capture display fields at enqueue time so the status panel # can render them without a DB lookup later. @@ -11150,6 +9783,7 @@ def reorganize_album_files(album_id): artist_id=meta['artist_id'], artist_name=meta['artist_name'], source=chosen_source, + metadata_source=metadata_source, ) return jsonify({"success": True, **result}) except Exception as e: @@ -11167,20 +9801,29 @@ def reorganize_all_artist_albums(artist_id): source (optional): same pick applied to every album. Per-album overrides aren't supported here — use the per-album modal for that. + mode (optional): 'api' or 'tags' applied to every album, same + shape as the per-album endpoint. """ try: from core.reorganize_queue import get_queue data = request.get_json() or {} chosen_source = data.get('source') or None + metadata_source = data.get('mode') or config_manager.get( + 'library.reorganize_metadata_source', 'api' + ) or 'api' + if metadata_source not in ('api', 'tags'): + metadata_source = 'api' albums = get_database().get_artist_albums_for_reorganize(artist_id) if not albums: return jsonify({"success": False, "error": "No albums found for this artist"}), 404 - # Apply the user's chosen source to every album, then hand off - # to the queue's bulk-enqueue helper which owns the loop+tally. + # Apply the user's chosen source + mode to every album, then + # hand off to the queue's bulk-enqueue helper which owns the + # loop+tally. for album in albums: album['source'] = chosen_source + album['metadata_source'] = metadata_source result = get_queue().enqueue_many(albums) return jsonify({ @@ -12012,6 +10655,7 @@ _SERVICE_ID_COLUMNS = { 'genius': {'artist': 'genius_id', 'track': 'genius_id'}, 'tidal': {'artist': 'tidal_id', 'album': 'tidal_id', 'track': 'tidal_id'}, 'qobuz': {'artist': 'qobuz_id', 'album': 'qobuz_id', 'track': 'qobuz_id'}, + 'amazon': {'artist': 'amazon_id', 'album': 'amazon_id', 'track': 'amazon_id'}, } @app.route('/api/library/manual-match', methods=['PUT']) @@ -12475,7 +11119,7 @@ def redownload_search_sources(track_id): quality = ext if ext in ('FLAC', 'MP3', 'OPUS', 'OGG', 'M4A', 'WAV') else candidate.quality or '' svc = source_name if source_name != 'default' else 'hybrid' uname = candidate.username - if uname in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): + if uname in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): svc = uname source_candidates.append({ 'username': uname, @@ -13373,7 +12017,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar logger.info(f"Processing enhanced album download for '{spotify_album['name']}' with {len(enhanced_tracks)} matched tracks") # Compute total_discs for multi-disc album subfolder support - total_discs = max((t['spotify_track'].get('disc_number', 1) for t in enhanced_tracks), default=1) + total_discs = max((t['spotify_track'].get('disc_number') or 1 for t in enhanced_tracks), default=1) spotify_album['total_discs'] = total_discs started_count = 0 @@ -13515,7 +12159,7 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): # Compute total_discs for multi-disc album subfolder support if official_spotify_tracks: - total_discs = max((t.get('disc_number', 1) for t in official_spotify_tracks), default=1) + total_discs = max((t.get('disc_number') or 1 for t in official_spotify_tracks), default=1) else: total_discs = 1 spotify_album['total_discs'] = total_discs @@ -15572,7 +14216,7 @@ def _pause_workers_for_scan(): 'mb': mb_worker, 'spotify': spotify_enrichment_worker, 'itunes': itunes_enrichment_worker, 'deezer': deezer_worker, 'audiodb': audiodb_worker, 'discogs': discogs_worker, 'lastfm': lastfm_worker, 'genius': genius_worker, 'tidal': tidal_enrichment_worker, 'qobuz': qobuz_enrichment_worker, - 'repair': repair_worker, 'soulid': soulid_worker, + 'amazon': amazon_worker, 'repair': repair_worker, 'soulid': soulid_worker, } for name, w in workers.items(): if w and hasattr(w, 'pause') and not getattr(w, 'paused', True): @@ -15588,7 +14232,7 @@ def _resume_workers_after_scan(): 'mb': mb_worker, 'spotify': spotify_enrichment_worker, 'itunes': itunes_enrichment_worker, 'deezer': deezer_worker, 'audiodb': audiodb_worker, 'discogs': discogs_worker, 'lastfm': lastfm_worker, 'genius': genius_worker, 'tidal': tidal_enrichment_worker, 'qobuz': qobuz_enrichment_worker, - 'repair': repair_worker, 'soulid': soulid_worker, + 'amazon': amazon_worker, 'repair': repair_worker, 'soulid': soulid_worker, } resumed = 0 for name, w in workers.items(): @@ -17781,7 +16425,7 @@ def _try_source_reuse(task_id, batch_id, track): if not source_tracks or not last_source: _sr.info("Skipped — no source_tracks or no last_source") return False - if last_source.get('username') in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): + if last_source.get('username') in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): _sr.info(f"Skipped — {last_source.get('username')} source (no folder-based reuse)") return False @@ -17883,7 +16527,7 @@ def _store_batch_source(batch_id, username, filename): """Browse the successful download's folder and store results on the batch for reuse.""" _sr = source_reuse_logger _sr.info(f"_store_batch_source called: batch={batch_id}, user={username}, file={filename}") - if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud'): + if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon'): _sr.info(f"Skipped — no batch_id or streaming source ({username})") return @@ -20033,6 +18677,9 @@ def get_spotify_album_tracks(album_id): client = _get_deezer_client() elif source_override == 'discogs': client = _get_discogs_client() + elif source_override == 'amazon': + from core.metadata.registry import get_amazon_client + client = get_amazon_client() elif source_override == 'musicbrainz': try: from core.musicbrainz_search import MusicBrainzSearchClient @@ -21053,6 +19700,26 @@ def deezer_download_test_download(): # =================================================================== +# AMAZON DOWNLOAD ENDPOINTS +# =================================================================== + +@app.route('/api/amazon/test-connection', methods=['GET']) +@admin_only +def amazon_test_connection(): + """Check whether the T2Tunes proxy is up and Amazon Music is reachable.""" + try: + from core.amazon_client import AmazonClient + c = AmazonClient() + status = c.status() + amazon_up = str(status.get('amazonMusic', '')).lower() == 'up' + return jsonify({ + 'connected': amazon_up, + 'status': status, + }) + except Exception as e: + return jsonify({'connected': False, 'error': str(e)}), 200 + + # TIDAL DOWNLOAD AUTH ENDPOINTS # =================================================================== @@ -25706,6 +24373,7 @@ def get_watchlist_artists(): "itunes_artist_id": artist.itunes_artist_id, # For iTunes-only artists "deezer_artist_id": getattr(artist, 'deezer_artist_id', None), "discogs_artist_id": getattr(artist, 'discogs_artist_id', None), + "amazon_artist_id": getattr(artist, 'amazon_artist_id', None), "include_albums": artist.include_albums, "include_eps": artist.include_eps, "include_singles": artist.include_singles, @@ -26505,10 +25173,10 @@ def watchlist_artist_config(artist_id): include_live, include_remixes, include_acoustic, include_compilations, artist_name, image_url, spotify_artist_id, itunes_artist_id, last_scan_timestamp, date_added, include_instrumentals, deezer_artist_id, - lookback_days, discogs_artist_id, preferred_metadata_source + lookback_days, discogs_artist_id, preferred_metadata_source, amazon_artist_id FROM watchlist_artists - WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? - """, (artist_id, artist_id, artist_id, artist_id)) + WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? OR amazon_artist_id = ? + """, (artist_id, artist_id, artist_id, artist_id, artist_id)) result = cursor.fetchone() conn.close() @@ -26517,10 +25185,11 @@ def watchlist_artist_config(artist_id): # Determine if this is an iTunes or Spotify artist is_itunes_artist = artist_id.isdigit() - spotify_id = result[9] # spotify_artist_id from query + spotify_id = result[9] # spotify_artist_id from query itunes_id = result[10] # itunes_artist_id from query deezer_id = result[14] # deezer_artist_id from query discogs_id = result[16] # discogs_artist_id from query + amazon_id = result[18] if len(result) > 18 else None # amazon_artist_id from query # Get artist info from Spotify (only for Spotify artists) artist_info = None @@ -26628,6 +25297,7 @@ def watchlist_artist_config(artist_id): "itunes_artist_id": itunes_id, "deezer_artist_id": deezer_id, "discogs_artist_id": discogs_id, + "amazon_artist_id": amazon_id, "watchlist_name": result[7], # Original stored watchlist artist name "global_metadata_source": get_primary_source(), }) @@ -26728,7 +25398,7 @@ def watchlist_artist_link_provider(artist_id): new_provider_id = data.get('provider_id', '').strip() provider = data.get('provider', '').strip() - valid_providers = ('spotify', 'itunes', 'deezer', 'discogs') + valid_providers = ('spotify', 'itunes', 'deezer', 'discogs', 'amazon') if provider not in valid_providers: return jsonify({"success": False, "error": f"Invalid provider. Must be one of: {', '.join(valid_providers)}"}), 400 @@ -26742,8 +25412,8 @@ def watchlist_artist_link_provider(artist_id): cursor.execute(""" SELECT id, artist_name, spotify_artist_id, itunes_artist_id FROM watchlist_artists - WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? - """, (artist_id, artist_id, artist_id, artist_id)) + WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? OR amazon_artist_id = ? + """, (artist_id, artist_id, artist_id, artist_id, artist_id)) row = cursor.fetchone() if not row: @@ -26754,7 +25424,7 @@ def watchlist_artist_link_provider(artist_id): artist_name = row[1] # Check for duplicate — another watchlist artist already has this provider ID - col_map = {'spotify': 'spotify_artist_id', 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id'} + col_map = {'spotify': 'spotify_artist_id', 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id', 'amazon': 'amazon_artist_id'} col = col_map[provider] if not is_clear: @@ -28195,6 +26865,118 @@ def get_discovery_shuffle(): logger.error(f"Error getting discovery shuffle playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 + +# ======================================================================== +# Personalized Playlists v2 — unified storage + manager-backed routes. +# Wraps every personalized playlist (Group A + Group B) behind one API +# surface. Generators in `core/personalized/generators/` register at +# import time; this set of routes exposes the manager for the UI. +# Legacy `/api/discover/personalized/...` endpoints stay alive for +# backward compat during the UI migration window. +# ======================================================================== + +# Trigger registration of every generator (side-effect import). +from core.personalized import generators as _personalized_generators # noqa: F401 +from core.personalized import api as _personalized_api +from core.personalized.manager import PersonalizedPlaylistManager as _PersonalizedManager + + +def _build_personalized_manager(): + """Construct a manager wired with whatever each generator needs. + + Per-request construction: the underlying services are cheap + accessors, so we don't bother caching. If profiling shows + overhead, this becomes a module-level lazy singleton.""" + from core.personalized_playlists import get_personalized_playlists_service + from core.seasonal_discovery import get_seasonal_discovery_service + database = get_database() + deps = types.SimpleNamespace( + database=database, + service=get_personalized_playlists_service(database, spotify_client), + seasonal_service=get_seasonal_discovery_service(spotify_client, database), + get_current_profile_id=get_current_profile_id, + get_active_discovery_source=_get_active_discovery_source, + ) + return _PersonalizedManager(database=database, deps=deps) + + +@app.route('/api/personalized/kinds', methods=['GET']) +def personalized_list_kinds(): + """List every registered personalized-playlist kind. Includes the + resolved variant list per kind that supports variants so the UI + can render kind+variant checkboxes without per-kind round-trips.""" + try: + manager = _build_personalized_manager() + return jsonify(_personalized_api.list_kinds(manager=manager)) + except Exception as e: + logger.error(f"Personalized kinds list error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/personalized/playlists', methods=['GET']) +def personalized_list_playlists(): + """List every persisted personalized playlist for the active profile.""" + try: + manager = _build_personalized_manager() + return jsonify(_personalized_api.list_playlists(manager, get_current_profile_id())) + except Exception as e: + logger.error(f"Personalized playlists list error: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/personalized/playlist/', methods=['GET']) +@app.route('/api/personalized/playlist//', methods=['GET']) +def personalized_get_playlist(kind, variant=''): + """Get one personalized playlist + its current track snapshot. + + Auto-creates the row from default config if it doesn't exist.""" + try: + manager = _build_personalized_manager() + return jsonify(_personalized_api.get_playlist_with_tracks( + manager, kind, variant, get_current_profile_id(), + )) + except ValueError as e: + return jsonify({"success": False, "error": str(e)}), 400 + except Exception as e: + logger.error(f"Personalized playlist get error ({kind}/{variant}): {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/personalized/playlist//refresh', methods=['POST']) +@app.route('/api/personalized/playlist///refresh', methods=['POST']) +def personalized_refresh_playlist(kind, variant=''): + """Run the kind's generator and persist the snapshot.""" + try: + manager = _build_personalized_manager() + body = request.get_json(silent=True) or {} + overrides = body.get('config_overrides') if isinstance(body.get('config_overrides'), dict) else None + return jsonify(_personalized_api.refresh_playlist( + manager, kind, variant, get_current_profile_id(), config_overrides=overrides, + )) + except ValueError as e: + return jsonify({"success": False, "error": str(e)}), 400 + except Exception as e: + logger.error(f"Personalized playlist refresh error ({kind}/{variant}): {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/personalized/playlist//config', methods=['PUT']) +@app.route('/api/personalized/playlist///config', methods=['PUT']) +def personalized_update_config(kind, variant=''): + """Patch the playlist's per-instance config.""" + try: + manager = _build_personalized_manager() + body = request.get_json(silent=True) or {} + return jsonify(_personalized_api.update_config( + manager, kind, variant, get_current_profile_id(), body, + )) + except ValueError as e: + return jsonify({"success": False, "error": str(e)}), 400 + except Exception as e: + logger.error(f"Personalized playlist config error ({kind}/{variant}): {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + @app.route('/api/discover/artist-blacklist', methods=['GET']) def get_discovery_artist_blacklist(): """Get all blacklisted discovery artists.""" @@ -33360,6 +32142,20 @@ except Exception as e: # END DEEZER INTEGRATION # ================================================================================================ +amazon_worker = None +try: + amazon_db = MusicDatabase() + amazon_worker = AmazonWorker(database=amazon_db) + amazon_worker.start() + if config_manager.get('amazon_enrichment_paused', False): + amazon_worker.pause() + logger.info("Amazon enrichment worker initialized (paused — restored from config)") + else: + logger.info("Amazon enrichment worker initialized and started") +except Exception as e: + logger.error(f"Amazon worker initialization failed: {e}") + amazon_worker = None + # ================================================================================================ # SPOTIFY ENRICHMENT INTEGRATION @@ -33630,6 +32426,7 @@ _init_service_search( qobuz_worker=qobuz_enrichment_worker, discogs_worker_obj=discogs_worker, audiodb_worker_obj=audiodb_worker, + amazon_worker_obj=amazon_worker, ) # Qobuz status / pause / resume routes are now served by the @@ -35024,6 +33821,11 @@ _register_enrichment_services([ config_paused_key='qobuz_enrichment_paused', extra_status_defaults={'authenticated': False}, ), + _EnrichmentService( + id='amazon', display_name='Amazon Music', + worker_getter=lambda: amazon_worker, + config_paused_key='amazon_enrichment_paused', + ), ]) _configure_enrichment_api( @@ -35043,7 +33845,7 @@ def _emit_rate_monitor_loop(): 'spotify': 'spotify_enrichment', 'itunes': 'itunes_enrichment', 'deezer': 'deezer_enrichment', 'lastfm': 'lastfm', 'genius': 'genius', 'musicbrainz': 'musicbrainz', 'audiodb': 'audiodb', 'discogs': 'discogs', - 'tidal': 'tidal_enrichment', 'qobuz': 'qobuz_enrichment', + 'tidal': 'tidal_enrichment', 'qobuz': 'qobuz_enrichment', 'amazon': 'amazon_enrichment', } while not globals().get('IS_SHUTTING_DOWN', False): @@ -35103,6 +33905,7 @@ def _emit_enrichment_status_loop(): 'genius-enrichment': lambda: genius_worker, 'tidal-enrichment': lambda: tidal_enrichment_worker, 'qobuz-enrichment': lambda: qobuz_enrichment_worker, + 'amazon-enrichment': lambda: amazon_worker, 'hydrabase': lambda: hydrabase_worker, 'soulid': lambda: soulid_worker, 'listening-stats': lambda: listening_stats_worker, diff --git a/webui/index.html b/webui/index.html index a315bd8b..474b5157 100644 --- a/webui/index.html +++ b/webui/index.html @@ -541,6 +541,29 @@ + +
+ +
+
+
Amazon Music Enrichment
+
+
Status: Idle +
+
No active matches +
+
Progress: 0 / 0 +
+
+
+
+
-
-

Service Status

-
-
-
- Metadata Source - ● -
-

Disconnected

-

Response: --

-
- -
-
-
-
- Media Server - ● -
-

Disconnected

-

Response: --

-
- -
-
-
-
- Download Source - ● -
-

Disconnected

-

Response: --

-
- -
-
-
- -
+ +
- -
-

Enrichment Services

-
- -
-
- - -
-
- -
- -
-
- -
-
-

Library

-

Checking status...

-
-
- - -
-
- -