diff --git a/core/import_resolution.py b/core/import_resolution.py new file mode 100644 index 00000000..8f9c712c --- /dev/null +++ b/core/import_resolution.py @@ -0,0 +1,412 @@ +"""Single-track import lookup and context-building helpers.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from utils.logging_config import get_logger + + +logger = get_logger("import_resolution") + + +def _get_metadata_service(): + from core import metadata_service + + return metadata_service + + +def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any: + if value is None: + return default + + for name in names: + if isinstance(value, dict): + if name in value and value[name] is not None: + return value[name] + else: + candidate = getattr(value, name, None) + if candidate is not None: + return candidate + return default + + +def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + normalized: List[Dict[str, Any]] = [] + for artist in artists: + if isinstance(artist, dict): + name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or '' + artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or '' + entry: Dict[str, Any] = {} + if name: + entry['name'] = str(name) + if artist_id: + entry['id'] = str(artist_id) + genres = _extract_lookup_value(artist, 'genres', default=None) + if genres is not None: + entry['genres'] = genres + if entry: + normalized.append(entry) + continue + + name = str(artist).strip() + if name: + normalized.append({'name': name}) + + return normalized + + +def _get_source_chain_for_lookup( + source_override: Optional[str] = None, + allow_fallback: bool = True, +) -> List[str]: + metadata_service = _get_metadata_service() + primary_source = metadata_service.get_primary_source() + source_chain = list(metadata_service.get_source_priority(primary_source)) + override = (source_override or '').strip().lower() + + if override: + source_chain = [override] + [source for source in source_chain if source != override] + + if not allow_fallback: + source_chain = source_chain[:1] + + return source_chain + + +def _build_track_search_query(source: str, title: str, artist: str) -> str: + base_query = " ".join(part for part in (title, artist) if part).strip() + if source == 'deezer' and title: + if artist: + return f'artist:"{artist}" track:"{title}"' + return f'track:"{title}"' + return base_query or title or artist + + +def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]: + if not search_results: + return None + + target_title = str(title or '').strip().lower() + target_artist = str(artist or '').strip().lower() + + for candidate in search_results: + candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower() + if candidate_title != target_title: + continue + + if not target_artist: + return candidate + + candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[])) + candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else '' + if candidate_artist_name == target_artist: + return candidate + + return search_results[0] + + +def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: + if not client or not hasattr(client, 'search_tracks'): + return [] + + try: + kwargs = {'limit': limit} + if source == 'spotify': + kwargs['allow_fallback'] = False + return client.search_tracks(query, **kwargs) or [] + except Exception as exc: + logger.debug("Could not search %s for %s: %s", source, query, exc) + return [] + + +def _build_single_import_context_payload( + track_data: Any, + source: Optional[str], + source_priority: List[str], + requested_title: str = '', + requested_artist: str = '', +) -> Dict[str, Any]: + album_data = _extract_lookup_value(track_data, 'album', default=None) + + track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '') + track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track' + track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[])) + if not track_artists and requested_artist: + track_artists = [{'name': requested_artist}] + + primary_track_artist = track_artists[0] if track_artists else {} + primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist' + primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '') + + album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or '' + album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '') + release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '') + album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album') + total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0) + album_images: List[Dict[str, Any]] = [] + album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[])) + + if isinstance(album_data, dict): + album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name + album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) + release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) + album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) + total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) + album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] + if not album_image_url: + album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + elif album_data: + album_name = album_name or str(album_data) + + if not album_artists and primary_artist_name: + album_artists = [{'name': primary_artist_name}] + + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + + track_info = { + 'id': track_id, + 'name': track_name, + 'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1), + 'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1), + 'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0), + 'artists': track_artists or [{'name': primary_artist_name}], + 'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''), + 'album': album_name, + 'album_id': album_id, + 'album_type': album_type, + 'release_date': release_date, + '_source': source or '', + } + + album_payload = { + 'id': album_id, + 'name': album_name, + 'release_date': release_date, + 'total_tracks': total_tracks or 1, + 'album_type': album_type, + 'image_url': album_image_url, + 'images': album_images, + 'artists': album_artists, + '_source': source or '', + } + + artist_payload = { + 'id': primary_artist_id, + 'name': primary_artist_name, + 'genres': [], + '_source': source or '', + } + + original_search = { + 'title': track_name, + 'artist': primary_artist_name, + 'album': album_name, + 'track_number': track_info['track_number'], + 'disc_number': track_info['disc_number'], + 'clean_title': track_name, + 'clean_album': album_name, + 'clean_artist': primary_artist_name, + 'artists': track_info['artists'], + 'duration_ms': track_info['duration_ms'], + 'id': track_id, + '_source': source or '', + } + + return { + 'success': bool(track_id or track_name != requested_title or album_name), + 'source': source, + 'source_priority': source_priority, + 'context': { + 'artist': artist_payload, + 'album': album_payload, + 'track_info': track_info, + 'original_search_result': original_search, + 'is_album_download': False, + 'has_clean_metadata': bool(track_id), + 'has_full_metadata': bool(track_id), + 'source': source, + 'source_priority': source_priority, + }, + } + + +def _build_single_import_fallback_context( + requested_title: str, + requested_artist: str, + source_priority: List[str], +) -> Dict[str, Any]: + artist_name = requested_artist or 'Unknown Artist' + title = requested_title or 'Unknown Track' + return { + 'success': False, + 'source': None, + 'source_priority': source_priority, + 'context': { + 'artist': { + 'id': '', + 'name': artist_name, + 'genres': [], + '_source': '', + }, + 'album': { + 'id': '', + 'name': '', + 'release_date': '', + 'total_tracks': 1, + 'album_type': 'album', + 'image_url': '', + 'images': [], + 'artists': [], + '_source': '', + }, + 'track_info': { + 'id': '', + 'name': title, + 'track_number': 1, + 'disc_number': 1, + 'duration_ms': 0, + 'artists': [{'name': artist_name}], + 'uri': '', + 'album': '', + 'album_id': '', + 'album_type': 'album', + 'release_date': '', + '_source': '', + }, + 'original_search_result': { + 'title': title, + 'artist': artist_name, + 'album': '', + 'track_number': 1, + 'disc_number': 1, + 'clean_title': title, + 'clean_album': '', + 'clean_artist': artist_name, + 'artists': [{'name': artist_name}], + 'duration_ms': 0, + 'id': '', + '_source': '', + }, + 'is_album_download': False, + 'has_clean_metadata': False, + 'has_full_metadata': False, + 'source': None, + 'source_priority': source_priority, + }, + } + + +def get_single_track_import_context( + title: str, + artist: str = '', + override_id: Optional[str] = None, + override_source: str = 'spotify', + source_override: Optional[str] = None, +) -> Dict[str, Any]: + """Build an import context for singles using source-priority metadata lookup.""" + metadata_service = _get_metadata_service() + source_priority = _get_source_chain_for_lookup(source_override=source_override, allow_fallback=True) + title = (title or '').strip() + artist = (artist or '').strip() + + if override_id: + chosen_source = (override_source or 'spotify').strip().lower() or 'spotify' + client = metadata_service.get_client_for_source(chosen_source) + if client and hasattr(client, 'get_track_details'): + try: + track_data = client.get_track_details(str(override_id)) + if track_data: + payload = _build_single_import_context_payload( + track_data, + chosen_source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + except Exception as exc: + logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc) + + for source in source_priority: + client = metadata_service.get_client_for_source(source) + if not client: + continue + + search_query = _build_track_search_query(source, title, artist) + if not search_query: + continue + + search_results = _search_tracks_for_source(source, client, search_query, limit=5) + if not search_results and search_query != title: + search_results = _search_tracks_for_source(source, client, title, limit=5) + if not search_results and artist and search_query != artist: + search_results = _search_tracks_for_source(source, client, artist, limit=5) + + if not search_results: + continue + + best_match = _pick_best_track_match(search_results, title or search_query, artist) + if not best_match: + continue + + resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '') + resolved_data = best_match + if resolved_track_id and hasattr(client, 'get_track_details'): + try: + detailed = client.get_track_details(resolved_track_id) + if detailed: + resolved_data = detailed + except Exception as exc: + logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc) + + payload = _build_single_import_context_payload( + resolved_data, + source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + + return _build_single_import_fallback_context(title, artist, source_priority) diff --git a/core/metadata_artwork.py b/core/metadata_artwork.py new file mode 100644 index 00000000..4c83f325 --- /dev/null +++ b/core/metadata_artwork.py @@ -0,0 +1,151 @@ +"""Album artwork helpers for metadata enrichment.""" + +from __future__ import annotations + +import os +import re +import urllib.request + +from core.import_context import get_import_context_album +from core.metadata_common import ( + _get_config_manager, + _get_image_dimensions, + _get_logger, + _get_mutagen_symbols, +) + + +def embed_album_art_metadata(audio_file, metadata: dict): + cfg = _get_config_manager() + logger_ = _get_logger() + symbols = _get_mutagen_symbols() + if not symbols: + return + + try: + image_data = None + mime_type = None + + release_mbid = metadata.get("musicbrainz_release_id") + if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False): + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if not image_data: + art_url = metadata.get("album_art_url") + if not art_url: + logger_.warning("No album art URL available for embedding.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + + if not image_data: + logger_.error("Failed to download album art data.") + return + + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data)) + elif isinstance(audio_file, symbols.FLAC): + picture = symbols.Picture() + picture.data = image_data + picture.type = 3 + picture.mime = mime_type + width, height = _get_image_dimensions(image_data) + picture.width = width or 640 + picture.height = height or 640 + picture.depth = 24 + audio_file.add_picture(picture) + elif isinstance(audio_file, symbols.MP4): + fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG + audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] + + logger_.info("Album art successfully embedded.") + except Exception as exc: + logger_.error("Error embedding album art: %s", exc) + + +def download_cover_art(album_info: dict, target_dir: str, context: dict = None): + cfg = _get_config_manager() + logger_ = _get_logger() + if cfg.get("metadata_enhancement.cover_art_download", True) is False: + return + + try: + cover_path = os.path.join(target_dir, "cover.jpg") + album_info = album_info or {} + release_mbid = album_info.get("musicbrainz_release_id") + prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False) + + if os.path.exists(cover_path): + if release_mbid and prefer_caa: + try: + existing_size = os.path.getsize(cover_path) + if existing_size > 200_000: + return + is_upgrade = True + except Exception: + return + else: + return + else: + is_upgrade = False + + image_data = None + if release_mbid and prefer_caa: + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if is_upgrade and not image_data: + logger_.error("CAA upgrade failed - keeping existing cover.jpg") + return + + if not image_data: + art_url = album_info.get("album_image_url") + if not art_url and context: + album_ctx = get_import_context_album(context) + art_url = album_ctx.get("image_url") + if not art_url and album_ctx.get("images"): + images = album_ctx.get("images", []) + if images and isinstance(images[0], dict): + art_url = images[0].get("url", "") + if art_url: + logger_.info("Using cover art URL from album context") + if art_url and "i.scdn.co" in art_url: + try: + from core.spotify_client import _upgrade_spotify_image_url + + art_url = _upgrade_spotify_image_url(art_url) + except Exception: + pass + elif art_url and "mzstatic.com" in art_url: + art_url = re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) + if not art_url: + logger_.warning("No cover art URL available for download.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + + if not image_data: + return + + with open(cover_path, "wb") as handle: + handle.write(image_data) + logger_.info("Cover art downloaded to: %s", cover_path) + except Exception as exc: + logger_.error("Error downloading cover.jpg: %s", exc) diff --git a/core/metadata_common.py b/core/metadata_common.py new file mode 100644 index 00000000..975a8e36 --- /dev/null +++ b/core/metadata_common.py @@ -0,0 +1,276 @@ +"""Shared low-level helpers for metadata enrichment.""" + +from __future__ import annotations + +import os +import threading +from types import SimpleNamespace +from typing import Any, Dict + +from utils.logging_config import get_logger + + +logger = get_logger("metadata_enrichment") + +_FILE_LOCKS: Dict[str, threading.Lock] = {} +_FILE_LOCKS_LOCK = threading.Lock() + + +class _NullConfigManager: + def get(self, _key: str, default: Any = None) -> Any: + return default + + +def _get_logger(): + return logger + + +def _get_config_manager(): + try: + from config.settings import config_manager as settings_config_manager + + return settings_config_manager + except Exception: + return _NullConfigManager() + + +def _get_database(): + try: + from database.music_database import get_database + + return get_database() + except Exception: + return None + + +def _get_itunes_client(): + try: + from core.metadata_service import get_itunes_client + + return get_itunes_client() + except Exception: + return None + + +def _extract_artist_name(artist: Any) -> str: + if isinstance(artist, dict): + return str(artist.get("name", "") or "") + if hasattr(artist, "name"): + return str(getattr(artist, "name") or "") + return str(artist) if artist else "" + + +def _get_mutagen_symbols(): + """Lazy mutagen import so tests can monkeypatch this without the package installed.""" + try: + from mutagen import File as MutagenFile + from mutagen.apev2 import APEv2, APENoHeaderError + from mutagen.flac import FLAC, Picture + from mutagen.id3 import ( + APIC, + ID3, + TBPM, + TCOP, + TDOR, + TDRC, + TCON, + TIT2, + TALB, + TPE1, + TPE2, + TPOS, + TPUB, + TRCK, + TSRC, + TXXX, + UFID, + TMED, + ) + from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm + from mutagen.oggvorbis import OggVorbis + try: + from mutagen.oggopus import OggOpus + except Exception: + OggOpus = None + except Exception as exc: + logger.debug("Mutagen unavailable for metadata enrichment: %s", exc) + return None + + return SimpleNamespace( + File=MutagenFile, + APEv2=APEv2, + APENoHeaderError=APENoHeaderError, + FLAC=FLAC, + Picture=Picture, + ID3=ID3, + APIC=APIC, + TBPM=TBPM, + TCOP=TCOP, + TDOR=TDOR, + TDRC=TDRC, + TCON=TCON, + TIT2=TIT2, + TALB=TALB, + TPE1=TPE1, + TPE2=TPE2, + TPOS=TPOS, + TPUB=TPUB, + TRCK=TRCK, + TSRC=TSRC, + TXXX=TXXX, + UFID=UFID, + TMED=TMED, + MP4=MP4, + MP4Cover=MP4Cover, + MP4FreeForm=MP4FreeForm, + OggVorbis=OggVorbis, + OggOpus=OggOpus, + ) + + +def _get_file_lock(file_path: str) -> threading.Lock: + with _FILE_LOCKS_LOCK: + lock = _FILE_LOCKS.get(file_path) + if lock is None: + lock = threading.Lock() + _FILE_LOCKS[file_path] = lock + return lock + + +def _is_ogg_opus(audio_file: Any) -> bool: + return type(audio_file).__name__ == "OggOpus" + + +def _is_vorbis_like(audio_file: Any, symbols: Any) -> bool: + vorbis_classes = tuple( + cls for cls in ( + getattr(symbols, "FLAC", None), + getattr(symbols, "OggVorbis", None), + ) if cls is not None + ) + return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or _is_ogg_opus(audio_file) + + +def _save_audio_file(audio_file: Any, symbols: Any) -> None: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.save(v1=0, v2_version=4) + elif isinstance(audio_file, symbols.FLAC): + audio_file.save(deleteid3=True) + else: + audio_file.save() + + +def _get_image_dimensions(data: bytes): + try: + if data[:8] == b"\x89PNG\r\n\x1a\n": + import struct + + w, h = struct.unpack(">II", data[16:24]) + return w, h + if data[:2] == b"\xff\xd8": + import struct + + i = 2 + while i < len(data) - 9: + if data[i] != 0xFF: + break + marker = data[i + 1] + if marker in (0xC0, 0xC2): + h, w = struct.unpack(">HH", data[i + 5 : i + 9]) + return w, h + length = struct.unpack(">H", data[i + 2 : i + 4])[0] + i += 2 + length + except Exception: + pass + return None, None + + +def _strip_all_non_audio_tags(file_path: str) -> dict: + summary = {"apev2_stripped": False, "apev2_tag_count": 0} + if os.path.splitext(file_path)[1].lower() != ".mp3": + return summary + + symbols = _get_mutagen_symbols() + if not symbols: + return summary + + try: + apev2_tags = symbols.APEv2(file_path) + tag_count = len(apev2_tags) + tag_keys = list(apev2_tags.keys()) + apev2_tags.delete(file_path) + summary["apev2_stripped"] = True + summary["apev2_tag_count"] = tag_count + logger.info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10])) + except symbols.APENoHeaderError: + pass + except Exception as exc: + logger.error("Could not strip APEv2 tags (non-fatal): %s", exc) + return summary + + +def _verify_metadata_written(file_path: str) -> bool: + symbols = _get_mutagen_symbols() + if not symbols: + return False + + try: + check = symbols.File(file_path) + if check is None or check.tags is None: + logger.info("[VERIFY] Tags are None after save: %s", file_path) + return False + + title_found = False + artist_found = False + if isinstance(check.tags, symbols.ID3): + title_found = bool(check.tags.getall("TIT2")) + artist_found = bool(check.tags.getall("TPE1")) + try: + symbols.APEv2(file_path) + logger.info("[VERIFY] APEv2 tags still present after processing!") + return False + except symbols.APENoHeaderError: + pass + elif _is_vorbis_like(check, symbols): + title_found = bool(check.get("title")) + artist_found = bool(check.get("artist")) + elif isinstance(check, symbols.MP4): + title_found = bool(check.get("\xa9nam")) + artist_found = bool(check.get("\xa9ART")) + + if not title_found or not artist_found: + logger.warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found) + return False + + logger.info("[VERIFY] Metadata verified OK") + return True + except Exception as exc: + logger.error("[VERIFY] Verification error (non-fatal): %s", exc) + return False + + +def wipe_source_tags(file_path: str) -> bool: + try: + _strip_all_non_audio_tags(file_path) + symbols = _get_mutagen_symbols() + if not symbols: + return False + + audio = symbols.File(file_path) + if audio is None: + return False + if hasattr(audio, "clear_pictures"): + audio.clear_pictures() + if audio.tags is not None: + tag_count = len(audio.tags) + audio.tags.clear() + else: + audio.add_tags() + tag_count = 0 + _save_audio_file(audio, symbols) + if tag_count > 0: + logger.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path)) + return True + except Exception as exc: + logger.error("[Tag Wipe] Failed (non-fatal): %s", exc) + return False diff --git a/core/metadata_enrichment.py b/core/metadata_enrichment.py index 293af162..89651f1d 100644 --- a/core/metadata_enrichment.py +++ b/core/metadata_enrichment.py @@ -1,1350 +1,56 @@ -"""Source-aware metadata enrichment helpers for imported audio files.""" +"""Compatibility facade and orchestration for metadata enrichment.""" from __future__ import annotations -import json import os -import re -import threading -import time -import urllib.request -from pathlib import Path -from types import SimpleNamespace -from typing import Any, Dict, Optional -from core.import_context import ( - get_import_clean_album, - get_import_clean_artist, - get_import_clean_title, - get_import_context_album, - get_import_context_artist, - get_import_original_search, - get_import_source, - get_import_source_ids, - get_import_track_info, - get_source_tag_names, - normalize_import_context, +from core.metadata_artwork import ( + download_cover_art as _download_cover_art_impl, + embed_album_art_metadata, ) -from config.settings import config_manager -from core.metadata_service import get_itunes_client -from database.music_database import get_database -from utils.logging_config import get_logger - - -logger = get_logger("metadata_enrichment") - -_FILE_LOCKS: Dict[str, threading.Lock] = {} -_FILE_LOCKS_LOCK = threading.Lock() - -_MB_RELEASE_CACHE: Dict[tuple, str] = {} -_MB_RELEASE_CACHE_LOCK = threading.RLock() -_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} -_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() - -_EDITION_PAREN_RE = re.compile( - r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' - r'limited|bonus|platinum|gold|super\s*deluxe|standard)' - r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]', - re.IGNORECASE, -) -_EDITION_BARE_RE = re.compile( - r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' - r'limited|bonus|platinum|gold|super\s*deluxe|standard)' - r'(?:\s+(?:edition|version))?\s*$', - re.IGNORECASE, +from core.metadata_common import ( + _get_config_manager, + _get_file_lock, + _get_image_dimensions as _common_get_image_dimensions, + _get_logger, + _get_mutagen_symbols, + _is_ogg_opus as _common_is_ogg_opus, + _is_vorbis_like, + _save_audio_file, + _strip_all_non_audio_tags, + _verify_metadata_written, + wipe_source_tags as _common_wipe_source_tags, ) +from core.metadata_lyrics import generate_lrc_file as _generate_lrc_file_impl +from core.metadata_source import embed_source_ids, extract_source_metadata -class _NullConfigManager: - def get(self, _key: str, default: Any = None) -> Any: - return default - - -def _get_logger(runtime=None): - return logger - - -def _get_config_manager(runtime=None): - return config_manager - - -def _get_database(runtime=None): - try: - return get_database() - except Exception: - return None - - -def _get_itunes_client(runtime=None): - try: - return get_itunes_client() - except Exception: - worker = getattr(runtime, "itunes_enrichment_worker", None) - if worker and getattr(worker, "client", None): - return worker.client - return getattr(runtime, "itunes_client", None) - - -def _extract_artist_name(artist: Any) -> str: - if isinstance(artist, dict): - return str(artist.get("name", "") or "") - if hasattr(artist, "name"): - return str(getattr(artist, "name") or "") - return str(artist) if artist else "" - - -def _get_mutagen_symbols(runtime=None): - """Lazy mutagen import so tests can monkeypatch this without the package installed.""" - try: - from mutagen import File as MutagenFile - from mutagen.apev2 import APEv2, APENoHeaderError - from mutagen.flac import FLAC, Picture - from mutagen.id3 import ( - APIC, - ID3, - TBPM, - TCOP, - TDOR, - TDRC, - TCON, - TIT2, - TALB, - TPE1, - TPE2, - TPOS, - TPUB, - TRCK, - TSRC, - TXXX, - UFID, - TMED, - ) - from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm - from mutagen.oggvorbis import OggVorbis - try: - from mutagen.oggopus import OggOpus - except Exception: - OggOpus = None - except Exception as exc: - _get_logger(runtime).debug("Mutagen unavailable for metadata enrichment: %s", exc) - return None - - return SimpleNamespace( - File=MutagenFile, - APEv2=APEv2, - APENoHeaderError=APENoHeaderError, - FLAC=FLAC, - Picture=Picture, - ID3=ID3, - APIC=APIC, - TBPM=TBPM, - TCOP=TCOP, - TDOR=TDOR, - TDRC=TDRC, - TCON=TCON, - TIT2=TIT2, - TALB=TALB, - TPE1=TPE1, - TPE2=TPE2, - TPOS=TPOS, - TPUB=TPUB, - TRCK=TRCK, - TSRC=TSRC, - TXXX=TXXX, - UFID=UFID, - TMED=TMED, - MP4=MP4, - MP4Cover=MP4Cover, - MP4FreeForm=MP4FreeForm, - OggVorbis=OggVorbis, - OggOpus=OggOpus, - ) - - -def _get_file_lock(file_path: str) -> threading.Lock: - with _FILE_LOCKS_LOCK: - lock = _FILE_LOCKS.get(file_path) - if lock is None: - lock = threading.Lock() - _FILE_LOCKS[file_path] = lock - return lock - - -def _is_ogg_opus(audio_file: Any) -> bool: - return type(audio_file).__name__ == "OggOpus" - - -def _is_vorbis_like(audio_file: Any, symbols: Any) -> bool: - vorbis_classes = tuple( - cls for cls in ( - getattr(symbols, "FLAC", None), - getattr(symbols, "OggVorbis", None), - ) if cls is not None - ) - return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or _is_ogg_opus(audio_file) - - -def _save_audio_file(audio_file: Any, symbols: Any) -> None: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, symbols.FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - - -def _strip_all_non_audio_tags(file_path: str, runtime=None) -> dict: - summary = {"apev2_stripped": False, "apev2_tag_count": 0} - if os.path.splitext(file_path)[1].lower() != ".mp3": - return summary - - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return summary - - try: - apev2_tags = symbols.APEv2(file_path) - tag_count = len(apev2_tags) - tag_keys = list(apev2_tags.keys()) - apev2_tags.delete(file_path) - summary["apev2_stripped"] = True - summary["apev2_tag_count"] = tag_count - _get_logger(runtime).info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10])) - except symbols.APENoHeaderError: - pass - except Exception as exc: - _get_logger(runtime).error("Could not strip APEv2 tags (non-fatal): %s", exc) - return summary - - -def _verify_metadata_written(file_path: str, runtime=None) -> bool: - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return False - - try: - check = symbols.File(file_path) - if check is None or check.tags is None: - _get_logger(runtime).info("[VERIFY] Tags are None after save: %s", file_path) - return False - - title_found = False - artist_found = False - if isinstance(check.tags, symbols.ID3): - title_found = bool(check.tags.getall("TIT2")) - artist_found = bool(check.tags.getall("TPE1")) - try: - symbols.APEv2(file_path) - _get_logger(runtime).info("[VERIFY] APEv2 tags still present after processing!") - return False - except symbols.APENoHeaderError: - pass - elif _is_vorbis_like(check, symbols): - title_found = bool(check.get("title")) - artist_found = bool(check.get("artist")) - elif isinstance(check, symbols.MP4): - title_found = bool(check.get("\xa9nam")) - artist_found = bool(check.get("\xa9ART")) - - if not title_found or not artist_found: - _get_logger(runtime).warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found) - return False - - _get_logger(runtime).info("[VERIFY] Metadata verified OK") - return True - except Exception as exc: - _get_logger(runtime).error("[VERIFY] Verification error (non-fatal): %s", exc) - return False +logger = _get_logger() def _get_image_dimensions(data: bytes): - try: - if data[:8] == b"\x89PNG\r\n\x1a\n": - import struct + return _common_get_image_dimensions(data) - w, h = struct.unpack(">II", data[16:24]) - return w, h - if data[:2] == b"\xff\xd8": - import struct - i = 2 - while i < len(data) - 9: - if data[i] != 0xFF: - break - marker = data[i + 1] - if marker in (0xC0, 0xC2): - h, w = struct.unpack(">HH", data[i + 5 : i + 9]) - return w, h - length = struct.unpack(">H", data[i + 2 : i + 4])[0] - i += 2 + length - except Exception: - pass - return None, None +def _is_ogg_opus(audio_file): + return _common_is_ogg_opus(audio_file) -def extract_source_metadata(context: dict, artist: dict, album_info: dict, runtime=None) -> dict: - if album_info is None: - album_info = {} +def wipe_source_tags(file_path: str) -> bool: + return _common_wipe_source_tags(file_path) - cfg = _get_config_manager(runtime) - context = normalize_import_context(context) - original_search = get_import_original_search(context) - album_ctx = get_import_context_album(context) - track_info = get_import_track_info(context) - source = get_import_source(context) - source_ids = get_import_source_ids(context) - artist_dict = artist if isinstance(artist, dict) else { - "name": _extract_artist_name(artist), - "id": getattr(artist, "id", ""), - "genres": list(getattr(artist, "genres", []) or []), - } +def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: + return _generate_lrc_file_impl(file_path, context, artist, album_info) - metadata: Dict[str, Any] = { - "source": source, - "source_track_id": source_ids["track_id"], - "source_artist_id": source_ids["artist_id"], - "source_album_id": source_ids["album_id"], - } - metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "")) - if original_search.get("clean_title"): - _get_logger(runtime).info("Metadata: Using clean title: '%s'", metadata["title"]) - elif album_info.get("clean_track_name"): - _get_logger(runtime).info("Metadata: Using album info clean name: '%s'", metadata["title"]) - else: - _get_logger(runtime).warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) +def download_cover_art(album_info: dict, target_dir: str, context: dict = None): + return _download_cover_art_impl(album_info, target_dir, context) - artists = original_search.get("artists") - if isinstance(artists, list) and artists: - all_artists = [] - for artist_item in artists: - if isinstance(artist_item, dict) and artist_item.get("name"): - all_artists.append(artist_item["name"]) - elif isinstance(artist_item, str): - all_artists.append(artist_item) - else: - all_artists.append(str(artist_item)) - metadata["artist"] = ", ".join(all_artists) - _get_logger(runtime).info("Metadata: Using all artists: '%s'", metadata["artist"]) - else: - metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context) - _get_logger(runtime).info("Metadata: Using primary artist: '%s'", metadata["artist"]) - raw_album_artist = artist_dict.get("name", "") or metadata["artist"] - track_info_ctx = track_info or {} - explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None - album_artists_for_collab = None - - if isinstance(explicit_artist, dict) and explicit_artist.get("name"): - raw_album_artist = explicit_artist["name"] - album_artists_for_collab = [explicit_artist] - elif isinstance(explicit_artist, str) and explicit_artist: - raw_album_artist = explicit_artist - album_artists_for_collab = [{"name": explicit_artist}] - elif album_ctx and isinstance(album_ctx, dict): - album_artists = album_ctx.get("artists", []) - if album_artists: - first_album_artist = album_artists[0] - if isinstance(first_album_artist, dict) and first_album_artist.get("name"): - raw_album_artist = first_album_artist["name"] - elif isinstance(first_album_artist, str) and first_album_artist: - raw_album_artist = first_album_artist - album_artists_for_collab = album_artists - - collab_mode = cfg.get("file_organization.collab_artist_mode", "first") - if collab_mode == "first" and raw_album_artist: - context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or [] - if len(context_artists) > 1: - first = context_artists[0] - raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first) - elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist): - artist_id = str(artist_dict.get("id", "")) - if source == "itunes" and artist_id.isdigit(): - try: - itunes_client = _get_itunes_client(runtime) - if itunes_client and hasattr(itunes_client, "resolve_primary_artist"): - resolved = itunes_client.resolve_primary_artist(artist_id) - if resolved and resolved != raw_album_artist: - raw_album_artist = resolved - except Exception: - pass - metadata["album_artist"] = raw_album_artist - - if album_info.get("is_album"): - metadata["album"] = album_info.get("album_name", "Unknown Album") - metadata["track_number"] = album_info.get("track_number", 1) - metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1 - _get_logger(runtime).info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) - else: - if album_ctx and album_ctx.get("name"): - _get_logger(runtime).info("[SAFEGUARD] Using album context name instead of track title for album metadata") - metadata["album"] = album_ctx["name"] - metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1 - metadata["total_tracks"] = album_ctx.get("total_tracks", 1) - else: - metadata["album"] = metadata["title"] - metadata["track_number"] = 1 - metadata["total_tracks"] = 1 - - disc_num = original_search.get("disc_number") - if disc_num is None and album_info: - disc_num = album_info.get("disc_number") - metadata["disc_number"] = disc_num if disc_num is not None else 1 - - if album_ctx and album_ctx.get("release_date"): - metadata["date"] = album_ctx["release_date"][:4] - - genres = artist_dict.get("genres") or [] - if genres: - from core.genre_filter import filter_genres - - filtered = filter_genres(list(genres[:2]), cfg) - if filtered: - metadata["genre"] = ", ".join(filtered) - - metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None - if not metadata["album_art_url"] and album_ctx: - album_image = album_ctx.get("image_url") - if not album_image and album_ctx.get("images"): - first_image = album_ctx["images"][0] - album_image = first_image.get("url") if isinstance(first_image, dict) else None - metadata["album_art_url"] = album_image - - _get_logger(runtime).info( - "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", - metadata.get("title"), - metadata.get("artist"), - metadata.get("album_artist"), - metadata.get("album"), - metadata.get("track_number"), - metadata.get("total_tracks"), - metadata.get("disc_number"), - ) - - return metadata - - -def embed_album_art_metadata(audio_file, metadata: dict, runtime=None): - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return - - try: - image_data = None - mime_type = None - - release_mbid = metadata.get("musicbrainz_release_id") - if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False): - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or "image/jpeg" - if not image_data or len(image_data) <= 1000: - image_data = None - except Exception: - image_data = None - - if not image_data: - art_url = metadata.get("album_art_url") - if not art_url: - logger_.warning("No album art URL available for embedding.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or "image/jpeg" - - if not image_data: - logger_.error("Failed to download album art data.") - return - - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data)) - elif isinstance(audio_file, symbols.FLAC): - picture = symbols.Picture() - picture.data = image_data - picture.type = 3 - picture.mime = mime_type - width, height = _get_image_dimensions(image_data) - picture.width = width or 640 - picture.height = height or 640 - picture.depth = 24 - audio_file.add_picture(picture) - elif isinstance(audio_file, symbols.MP4): - fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG - audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] - - logger_.info("Album art successfully embedded.") - except Exception as exc: - logger_.error("Error embedding album art: %s", exc) - - -def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return - - try: - tag_config = { - "SPOTIFY_TRACK_ID": "spotify.tags.track_id", - "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", - "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", - "ITUNES_TRACK_ID": "itunes.tags.track_id", - "ITUNES_ARTIST_ID": "itunes.tags.artist_id", - "ITUNES_ALBUM_ID": "itunes.tags.album_id", - "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", - "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", - "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", - "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", - "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", - "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", - "RELEASETYPE": "musicbrainz.tags.release_type", - "ORIGINALDATE": "musicbrainz.tags.original_date", - "RELEASESTATUS": "musicbrainz.tags.release_status", - "RELEASECOUNTRY": "musicbrainz.tags.release_country", - "BARCODE": "musicbrainz.tags.barcode", - "MEDIA": "musicbrainz.tags.media", - "TOTALDISCS": "musicbrainz.tags.total_discs", - "CATALOGNUMBER": "musicbrainz.tags.catalog_number", - "SCRIPT": "musicbrainz.tags.script", - "ASIN": "musicbrainz.tags.asin", - "DEEZER_TRACK_ID": "deezer.tags.track_id", - "DEEZER_ARTIST_ID": "deezer.tags.artist_id", - "AUDIODB_TRACK_ID": "audiodb.tags.track_id", - "TIDAL_TRACK_ID": "tidal.tags.track_id", - "TIDAL_ARTIST_ID": "tidal.tags.artist_id", - "QOBUZ_TRACK_ID": "qobuz.tags.track_id", - "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", - "GENIUS_TRACK_ID": "genius.tags.track_id", - } - - def _tag_enabled(path: str) -> bool: - return cfg.get(path, True) is not False - - def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: - if not a or not b: - return False - from difflib import SequenceMatcher - - norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() - return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold - - context = normalize_import_context(context) - source = (metadata.get("source") or "").strip().lower() - source_ids = {} - if source: - source_tag_names = get_source_tag_names(source) - source_track_id = metadata.get("source_track_id") - source_artist_id = metadata.get("source_artist_id") - source_album_id = metadata.get("source_album_id") - if cfg.get(f"{source}.embed_tags", True) is not False: - if source_tag_names.get("track") and source_track_id: - source_ids[source_tag_names["track"]] = source_track_id - if source_tag_names.get("artist") and source_artist_id: - source_ids[source_tag_names["artist"]] = source_artist_id - if source_tag_names.get("album") and source_album_id: - source_ids[source_tag_names["album"]] = source_album_id - - if not source_ids: - if cfg.get("spotify.embed_tags", True) is not False: - if metadata.get("spotify_track_id"): - source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] - if metadata.get("spotify_artist_id"): - source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] - if metadata.get("spotify_album_id"): - source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] - if cfg.get("itunes.embed_tags", True) is not False: - if metadata.get("itunes_track_id"): - source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] - if metadata.get("itunes_artist_id"): - source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] - if metadata.get("itunes_album_id"): - source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] - - track_title = metadata.get("title", "") - artist_name = metadata.get("album_artist", "") or metadata.get("artist", "") - track_info = get_import_track_info(context) - explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None - batch_artist_name = None - if isinstance(explicit_artist, dict) and explicit_artist.get("name"): - batch_artist_name = explicit_artist["name"] - elif isinstance(explicit_artist, str) and explicit_artist: - batch_artist_name = explicit_artist - - pp = { - "id_tags": source_ids, - "track_title": track_title, - "artist_name": artist_name, - "batch_artist_name": batch_artist_name, - "metadata": metadata, - "recording_mbid": None, - "artist_mbid": None, - "release_mbid": "", - "mb_genres": [], - "isrc": None, - "deezer_bpm": None, - "deezer_isrc": None, - "audiodb_mood": None, - "audiodb_style": None, - "audiodb_genre": None, - "tidal_isrc": None, - "tidal_copyright": None, - "qobuz_isrc": None, - "qobuz_copyright": None, - "qobuz_label": None, - "lastfm_tags": [], - "lastfm_url": None, - "genius_url": None, - "release_year": None, - } - - source_order = cfg.get("metadata_enhancement.post_process_order", None) - if not isinstance(source_order, list) or not source_order: - source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] - - db = _get_database(runtime) - - for source_name in source_order: - if source_name == "musicbrainz": - if cfg.get("musicbrainz.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - mb_worker = getattr(runtime, "mb_worker", None) - mb_service = mb_worker.mb_service if mb_worker else None - if not mb_service: - continue - try: - result = mb_service.match_recording(track_title, artist_name) - if result and result.get("mbid"): - pp["recording_mbid"] = result["mbid"] - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] - details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) - if details: - isrcs = details.get("isrcs", []) - if isrcs: - pp["isrc"] = isrcs[0] - pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] - - track_artist_name = metadata.get("artist", "") or artist_name - if ", " in track_artist_name: - track_artist_name = track_artist_name.split(", ")[0] - artist_result = mb_service.match_artist(track_artist_name) - if artist_result and artist_result.get("mbid"): - pp["artist_mbid"] = artist_result["mbid"] - pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] - - album_name_for_mb = metadata.get("album", "") - if album_name_for_mb: - artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() - rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) - rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) - with _MB_RELEASE_CACHE_LOCK: - cached = _MB_RELEASE_CACHE.get(rc_key_norm) - if cached is None: - cached = _MB_RELEASE_CACHE.get(rc_key_exact) - if cached is not None: - pp["release_mbid"] = cached - else: - try: - rc_result = mb_service.match_release(album_name_for_mb, artist_name) - pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" - except Exception: - pp["release_mbid"] = "" - _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] - _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] - if pp["release_mbid"]: - pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] - - if pp["release_mbid"]: - with _MB_RELEASE_DETAIL_CACHE_LOCK: - release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) - if release_detail is None: - release_detail = mb_service.mb_client.get_release( - pp["release_mbid"], - includes=["release-groups", "labels", "media", "artist-credits", "recordings"], - ) or {} - with _MB_RELEASE_DETAIL_CACHE_LOCK: - _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail - if release_detail: - rg = release_detail.get("release-group", {}) - if rg.get("id"): - pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] - ac = release_detail.get("artist-credit", []) - if ac and isinstance(ac[0], dict): - aa = ac[0].get("artist", {}) - if aa.get("id"): - pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] - if rg.get("primary-type"): - pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] - if rg.get("first-release-date"): - pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] - if not pp["release_year"] and len(rg["first-release-date"]) >= 4: - year = rg["first-release-date"][:4] - if year.isdigit(): - pp["release_year"] = year - if release_detail.get("status"): - pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] - if release_detail.get("country"): - pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] - if release_detail.get("barcode"): - pp["id_tags"]["BARCODE"] = release_detail["barcode"] - media_list = release_detail.get("media", []) - if media_list: - fmt = media_list[0].get("format", "") - if fmt: - pp["id_tags"]["MEDIA"] = fmt - pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) - label_info = release_detail.get("label-info", []) - if label_info and isinstance(label_info[0], dict): - cat = label_info[0].get("catalog-number", "") - if cat: - pp["id_tags"]["CATALOGNUMBER"] = cat - text_rep = release_detail.get("text-representation", {}) - if isinstance(text_rep, dict) and text_rep.get("script"): - pp["id_tags"]["SCRIPT"] = text_rep["script"] - if release_detail.get("asin"): - pp["id_tags"]["ASIN"] = release_detail["asin"] - track_num = metadata.get("track_number") - disc_num = metadata.get("disc_number") or 1 - if track_num and media_list: - try: - track_num_int = int(track_num) - disc_num_int = int(disc_num) - for medium in media_list: - if medium.get("position", 1) == disc_num_int: - for mtrack in (medium.get("tracks") or medium.get("track-list", [])): - if mtrack.get("position") == track_num_int: - if mtrack.get("id"): - pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] - release_recording = mtrack.get("recording", {}) - if release_recording.get("id"): - pp["recording_mbid"] = release_recording["id"] - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] - break - break - except (ValueError, TypeError): - pass - except Exception as exc: - logger_.error("MusicBrainz lookup failed (non-fatal): %s", exc) - continue - - if source_name == "deezer": - if cfg.get("deezer.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - deezer_worker = getattr(runtime, "deezer_worker", None) - dz_client = deezer_worker.client if deezer_worker else None - if not dz_client: - continue - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): - dz_track_id = dz_result["id"] - pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) - dz_artist_id = dz_result.get("artist", {}).get("id") - if dz_artist_id: - pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get("bpm") - if bpm_val and bpm_val > 0: - pp["deezer_bpm"] = bpm_val - dz_isrc = dz_details.get("isrc") - if dz_isrc: - pp["deezer_isrc"] = dz_isrc - if not pp["release_year"]: - dz_album = dz_result.get("album", {}) - dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" - if len(dz_release) >= 4 and dz_release[:4].isdigit(): - pp["release_year"] = dz_release[:4] - except Exception as exc: - logger_.error("Deezer lookup failed (non-fatal): %s", exc) - continue - - if source_name == "audiodb": - if cfg.get("audiodb.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - audiodb_worker = getattr(runtime, "audiodb_worker", None) - adb_client = audiodb_worker.client if audiodb_worker else None - if not adb_client: - continue - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): - adb_track_id = adb_result.get("idTrack") - if adb_track_id: - pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) - adb_mb_track = adb_result.get("strMusicBrainzID") - if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track - pp["recording_mbid"] = adb_mb_track - adb_mb_artist = adb_result.get("strMusicBrainzArtistID") - if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: - pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist - pp["artist_mbid"] = adb_mb_artist - pp["audiodb_mood"] = adb_result.get("strMood") or None - pp["audiodb_style"] = adb_result.get("strStyle") or None - pp["audiodb_genre"] = adb_result.get("strGenre") or None - except Exception as exc: - logger_.error("AudioDB lookup failed (non-fatal): %s", exc) - continue - - if source_name == "tidal": - if cfg.get("tidal.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - tidal_client = getattr(runtime, "tidal_client", None) - if not (tidal_client and tidal_client.is_authenticated()): - continue - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get("title", ""), track_title): - td_track_id = td_result.get("id") - if td_track_id: - pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) - td_artist = td_result.get("artist", {}) - if isinstance(td_artist, dict) and td_artist.get("id"): - pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - pp["tidal_isrc"] = td_details.get("isrc") - td_copyright = td_details.get("copyright") - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get("text", td_copyright.get("name", "")) - pp["tidal_copyright"] = td_copyright or None - if not pp["release_year"]: - td_album = td_result.get("album", {}) - td_release = "" - if isinstance(td_album, dict): - td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") - if len(td_release) >= 4 and td_release[:4].isdigit(): - pp["release_year"] = td_release[:4] - except Exception as exc: - logger_.error("Tidal lookup failed (non-fatal): %s", exc) - continue - - if source_name == "qobuz": - if cfg.get("qobuz.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) - qz_client = qobuz_worker.client if qobuz_worker else None - if not (qz_client and qz_client.is_authenticated()): - continue - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = qz_result.get("performer") or {} - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get("name", "") - if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get("id") - if qz_track_id: - pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) - if qz_performer.get("id"): - pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) - qz_isrc = qz_result.get("isrc") - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) - if qz_isrc: - pp["qobuz_isrc"] = qz_isrc - qz_copyright = qz_result.get("copyright") - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) - if isinstance(qz_copyright, str): - pp["qobuz_copyright"] = qz_copyright - qz_album = qz_result.get("album", {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get("label", {}) - if isinstance(qz_label_info, dict) and qz_label_info.get("name"): - pp["qobuz_label"] = qz_label_info["name"] - if not pp["release_year"]: - qz_release = str(qz_album.get("release_date_original", "") or "") - if not qz_release: - qz_ts = qz_album.get("released_at") - if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: - import datetime as _dt - qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) - if len(qz_release) >= 4 and qz_release[:4].isdigit(): - pp["release_year"] = qz_release[:4] - except Exception as exc: - logger_.error("Qobuz lookup failed (non-fatal): %s", exc) - continue - - if source_name == "lastfm": - if cfg.get("lastfm.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - lastfm_worker = getattr(runtime, "lastfm_worker", None) - lf_client = lastfm_worker.client if lastfm_worker else None - if not lf_client: - continue - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get("url") - if lf_url: - pp["lastfm_url"] = lf_url - lf_toptags = lf_result.get("toptags", {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get("tag", []) - if isinstance(tag_list, list): - pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] - elif isinstance(tag_list, dict) and tag_list.get("name"): - pp["lastfm_tags"] = [tag_list["name"]] - except Exception as exc: - logger_.error("Last.fm lookup failed (non-fatal): %s", exc) - continue - - if source_name == "genius": - if cfg.get("genius.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - import core.genius_client as _genius_module - - if time.time() < _genius_module._rate_limit_until: - logger_.info("Genius rate-limited, skipping (non-blocking)") - continue - genius_worker = getattr(runtime, "genius_worker", None) - g_client = genius_worker.client if genius_worker else None - if not g_client: - continue - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get("id") - if g_id: - pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) - g_url = g_result.get("url") - if g_url: - pp["genius_url"] = g_url - except Exception as exc: - logger_.error("Genius lookup failed (non-fatal): %s", exc) - continue - - if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: - return - - filtered_tags: Dict[str, str] = {} - for tag_name, value in pp["id_tags"].items(): - config_path = tag_config.get(tag_name) - if config_path and not _tag_enabled(config_path): - continue - filtered_tags[tag_name] = value - - written = [] - id3_tag_map = { - "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), - "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), - "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), - "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), - "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), - "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), - "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), - "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), - "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), - "ORIGINALDATE": ("TDOR", None), - "MEDIA": ("TMED", None), - } - vorbis_tag_map = { - "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", - "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", - "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", - "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", - "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", - "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", - } - mp4_tag_map = { - "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", - "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", - "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", - "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", - "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", - "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", - "RELEASETYPE": "MusicBrainz Album Type", - "RELEASESTATUS": "MusicBrainz Album Status", - "RELEASECOUNTRY": "MusicBrainz Album Release Country", - } - - if isinstance(audio_file.tags, symbols.ID3): - for tag_name, value in filtered_tags.items(): - spec = id3_tag_map.get(tag_name) - if spec: - frame_type, desc = spec - if frame_type == "UFID": - audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) - written.append(f"UFID:{desc}") - elif frame_type == "TDOR": - audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) - written.append("TDOR") - elif frame_type == "TMED": - audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) - written.append("TMED") - else: - audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) - written.append(f"TXXX:{desc}") - else: - audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) - written.append(f"TXXX:{tag_name}") - elif _is_vorbis_like(audio_file, symbols): - for tag_name, value in filtered_tags.items(): - audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] - written.append(vorbis_tag_map.get(tag_name, tag_name)) - elif isinstance(audio_file, symbols.MP4): - for tag_name, value in filtered_tags.items(): - key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" - audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] - written.append(key) - - if written: - logger_.info("Embedded IDs: %s", ", ".join(written)) - - release_year = pp["release_year"] - needs_date_tag = bool(release_year and not metadata.get("date")) - if needs_date_tag: - metadata["date"] = release_year - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["date"] = [release_year] - elif isinstance(audio_file, symbols.MP4): - audio_file["\xa9day"] = [release_year] - logger_.info("Date tag: %s", release_year) - - if _tag_enabled("deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: - bpm_int = int(pp["deezer_bpm"]) - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["BPM"] = [str(bpm_int)] - elif isinstance(audio_file, symbols.MP4): - audio_file["tmpo"] = [bpm_int] - logger_.info("BPM: %s", bpm_int) - - if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["MOOD"] = [pp["audiodb_mood"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] - - if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["STYLE"] = [pp["audiodb_style"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] - - if _tag_enabled("metadata_enhancement.tags.genre_merge"): - enrichment_genres = [] - if _tag_enabled("musicbrainz.tags.genres"): - enrichment_genres += pp["mb_genres"] - if pp["audiodb_genre"] and _tag_enabled("audiodb.tags.genre"): - enrichment_genres.append(pp["audiodb_genre"]) - if _tag_enabled("lastfm.tags.genres"): - enrichment_genres += pp["lastfm_tags"] - if enrichment_genres: - from core.genre_filter import filter_genres as _filter_genres - - enrichment_genres = _filter_genres(enrichment_genres, cfg) - source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] - seen = set() - merged = [] - for genre in source_genres + enrichment_genres: - key = genre.strip().lower() - if key and key not in seen: - seen.add(key) - merged.append(genre.strip().title()) - if len(merged) >= 5: - break - if merged: - genre_string = ", ".join(merged) - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["GENRE"] = [genre_string] - elif isinstance(audio_file, symbols.MP4): - audio_file["\xa9gen"] = [genre_string] - logger_.info("Genres merged: %s", genre_string) - - isrc_candidates = [] - if pp["isrc"] and _tag_enabled("musicbrainz.tags.isrc"): - isrc_candidates.append(("MusicBrainz", pp["isrc"])) - if pp["deezer_isrc"] and _tag_enabled("deezer.tags.isrc"): - isrc_candidates.append(("Deezer", pp["deezer_isrc"])) - if pp["tidal_isrc"] and _tag_enabled("tidal.tags.isrc"): - isrc_candidates.append(("Tidal", pp["tidal_isrc"])) - if pp["qobuz_isrc"] and _tag_enabled("qobuz.tags.isrc"): - isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) - if isrc_candidates: - isrc_source, final_isrc = isrc_candidates[0] - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["ISRC"] = [final_isrc] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] - logger_.info("ISRC (%s): %s", isrc_source, final_isrc) - - copyright_candidates = [] - if pp["tidal_copyright"] and _tag_enabled("tidal.tags.copyright"): - copyright_candidates.append(("Tidal", pp["tidal_copyright"])) - if pp["qobuz_copyright"] and _tag_enabled("qobuz.tags.copyright"): - copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) - if copyright_candidates: - copyright_source, final_copyright = copyright_candidates[0] - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["COPYRIGHT"] = [final_copyright] - elif isinstance(audio_file, symbols.MP4): - audio_file["cprt"] = [final_copyright] - logger_.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) - - if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["LABEL"] = [pp["qobuz_label"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] - - if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["LASTFM_URL"] = [pp["lastfm_url"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] - - if _tag_enabled("genius.tags.url") and pp["genius_url"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["GENIUS_URL"] = [pp["genius_url"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] - - release_id = pp["release_mbid"] - if release_id: - metadata["musicbrainz_release_id"] = release_id - if db is not None: - try: - album_name_for_db = metadata.get("album", "") - album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") - if album_name_for_db and album_artist_for_db: - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute( - """ - UPDATE albums SET year = ? - WHERE (year IS NULL OR year = 0) - AND id IN ( - SELECT al.id FROM albums al - JOIN artists ar ON ar.id = al.artist_id - WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) - ) - """, - (int(release_year), album_name_for_db, album_artist_for_db), - ) - if cursor.rowcount > 0: - conn.commit() - logger_.info("Updated album year to %s in database", release_year) - else: - conn.rollback() - finally: - conn.close() - except Exception as exc: - logger_.error("Could not update album year in DB: %s", exc) - - except Exception as exc: - logger_.error("Error embedding source IDs (non-fatal): %s", exc) - - -def download_cover_art(album_info: dict, target_dir: str, context: dict = None, runtime=None): - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - if cfg.get("metadata_enhancement.cover_art_download", True) is False: - return - - try: - cover_path = os.path.join(target_dir, "cover.jpg") - album_info = album_info or {} - release_mbid = album_info.get("musicbrainz_release_id") - prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False) - - if os.path.exists(cover_path): - if release_mbid and prefer_caa: - try: - existing_size = os.path.getsize(cover_path) - if existing_size > 200_000: - return - is_upgrade = True - except Exception: - return - else: - return - else: - is_upgrade = False - - image_data = None - if release_mbid and prefer_caa: - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - if not image_data or len(image_data) <= 1000: - image_data = None - except Exception: - image_data = None - - if is_upgrade and not image_data: - logger_.error("CAA upgrade failed - keeping existing cover.jpg") - return - - if not image_data: - art_url = album_info.get("album_image_url") - if not art_url and context: - album_ctx = get_import_context_album(context) - art_url = album_ctx.get("image_url") - if not art_url and album_ctx.get("images"): - images = album_ctx.get("images", []) - if images and isinstance(images[0], dict): - art_url = images[0].get("url", "") - if art_url: - logger_.info("Using cover art URL from album context") - if art_url and "i.scdn.co" in art_url: - try: - from core.spotify_client import _upgrade_spotify_image_url - - art_url = _upgrade_spotify_image_url(art_url) - except Exception: - pass - elif art_url and "mzstatic.com" in art_url: - import re as _re - - art_url = _re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) - if not art_url: - logger_.warning("No cover art URL available for download.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - - if not image_data: - return - - with open(cover_path, "wb") as handle: - handle.write(image_data) - logger_.info("Cover art downloaded to: %s", cover_path) - except Exception as exc: - logger_.error("Error downloading cover.jpg: %s", exc) - - -def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: - return False - - try: - from core.lyrics_client import lyrics_client - - context = normalize_import_context(context) - original_search = get_import_original_search(context) - album_context = get_import_context_album(context) - track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) - - if isinstance(artist, dict): - artist_name = artist.get("name", "Unknown Artist") - elif hasattr(artist, "name"): - artist_name = artist.name - else: - artist_name = str(artist) if artist else "Unknown Artist" - - album_name = None - duration_seconds = None - if album_info and album_info.get("is_album"): - album_name = ( - get_import_clean_album(context, album_info=album_info, default="") - or album_info.get("album_name") - or album_context.get("name") - ) - - if original_search.get("duration_ms"): - duration_seconds = int(original_search["duration_ms"] / 1000) - - success = lyrics_client.create_lrc_file( - audio_file_path=file_path, - track_name=track_name, - artist_name=artist_name, - album_name=album_name, - duration_seconds=duration_seconds, - ) - - if success: - logger_.info("LRC file generated for: %s", track_name) - else: - logger_.warning("No lyrics found for: %s", track_name) - return success - except Exception as exc: - logger_.error("Error generating LRC file for %s: %s", file_path, exc) - return False - - -def wipe_source_tags(file_path: str, runtime=None) -> bool: - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - _ = cfg # keep signature parallel with other helpers - - try: - _strip_all_non_audio_tags(file_path, runtime=runtime) - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return False - - audio = symbols.File(file_path) - if audio is None: - return False - if hasattr(audio, "clear_pictures"): - audio.clear_pictures() - if audio.tags is not None: - tag_count = len(audio.tags) - audio.tags.clear() - else: - audio.add_tags() - tag_count = 0 - _save_audio_file(audio, symbols) - if tag_count > 0: - logger_.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path)) - return True - except Exception as exc: - logger_.error("[Tag Wipe] Failed (non-fatal): %s", exc) - return False - - -def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) +def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: + cfg = _get_config_manager() + logger_ = _get_logger() if cfg.get("metadata_enhancement.enabled", True) is False: logger_.warning("Metadata enhancement disabled in config.") return True @@ -1352,7 +58,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf if album_info is None: album_info = {} - symbols = _get_mutagen_symbols(runtime) + symbols = _get_mutagen_symbols() if not symbols: logger_.error("Mutagen is unavailable, cannot enhance metadata.") return False @@ -1361,7 +67,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf with file_lock: logger_.info("Enhancing metadata for: %s", os.path.basename(file_path)) try: - _strip_all_non_audio_tags(file_path, runtime=runtime) + _strip_all_non_audio_tags(file_path) audio_file = symbols.File(file_path) if audio_file is None: logger_.error("Could not load audio file with Mutagen: %s", file_path) @@ -1380,7 +86,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf _save_audio_file(audio_file, symbols) - metadata = extract_source_metadata(context, artist, album_info, runtime=runtime) + metadata = extract_source_metadata(context, artist, album_info) if not metadata: logger_.error("Could not extract source metadata, saving with cleared tags.") _save_audio_file(audio_file, symbols) @@ -1443,13 +149,13 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf if metadata.get("disc_number"): audio_file["disk"] = [(metadata["disc_number"], 0)] - embed_source_ids(audio_file, metadata, context, runtime=runtime) + embed_source_ids(audio_file, metadata, context) if album_info is not None and metadata.get("musicbrainz_release_id"): album_info["musicbrainz_release_id"] = metadata["musicbrainz_release_id"] if cfg.get("metadata_enhancement.embed_album_art", True): - embed_album_art_metadata(audio_file, metadata, runtime=runtime) + embed_album_art_metadata(audio_file, metadata) quality = context.get("_audio_quality", "") if quality and cfg.get("metadata_enhancement.tags.quality_tag", True) is not False: @@ -1462,7 +168,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf _save_audio_file(audio_file, symbols) - verified = _verify_metadata_written(file_path, runtime=runtime) + verified = _verify_metadata_written(file_path) if verified: logger_.info("Metadata enhanced successfully.") else: @@ -1478,9 +184,3 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf logger_.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None") logger_.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc()) return False - - -def _normalize_album_cache_key(album_name: str) -> str: - result = _EDITION_PAREN_RE.sub("", album_name or "") - result = _EDITION_BARE_RE.sub("", result) - return result.lower().strip() diff --git a/core/metadata_lyrics.py b/core/metadata_lyrics.py new file mode 100644 index 00000000..99a7612b --- /dev/null +++ b/core/metadata_lyrics.py @@ -0,0 +1,63 @@ +"""Lyrics export helpers for metadata enrichment.""" + +from __future__ import annotations + +from core.import_context import ( + get_import_clean_album, + get_import_clean_title, + get_import_context_album, + get_import_original_search, + normalize_import_context, +) +from core.metadata_common import _get_config_manager, _get_logger + + +def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: + cfg = _get_config_manager() + logger_ = _get_logger() + if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: + return False + + try: + from core.lyrics_client import lyrics_client + + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_context = get_import_context_album(context) + track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) + + if isinstance(artist, dict): + artist_name = artist.get("name", "Unknown Artist") + elif hasattr(artist, "name"): + artist_name = artist.name + else: + artist_name = str(artist) if artist else "Unknown Artist" + + album_name = None + duration_seconds = None + if album_info and album_info.get("is_album"): + album_name = ( + get_import_clean_album(context, album_info=album_info, default="") + or album_info.get("album_name") + or album_context.get("name") + ) + + if original_search.get("duration_ms"): + duration_seconds = int(original_search["duration_ms"] / 1000) + + success = lyrics_client.create_lrc_file( + audio_file_path=file_path, + track_name=track_name, + artist_name=artist_name, + album_name=album_name, + duration_seconds=duration_seconds, + ) + + if success: + logger_.info("LRC file generated for: %s", track_name) + else: + logger_.warning("No lyrics found for: %s", track_name) + return success + except Exception as exc: + logger_.error("Error generating LRC file for %s: %s", file_path, exc) + return False diff --git a/core/metadata_service.py b/core/metadata_service.py index 861f0b08..68cf0b92 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -1,5 +1,5 @@ """ -Metadata Service - Centralized metadata source selection +Metadata Service - Centralized metadata source selection and provider access. ALL metadata source decisions flow through this module. Other files import get_primary_source() and get_primary_client() instead of reimplementing @@ -100,6 +100,35 @@ def get_source_priority(preferred_source: str): return ordered +def _get_source_chain_for_lookup(options: MetadataLookupOptions) -> List[str]: + primary_source = get_primary_source() + source_chain = list(get_source_priority(primary_source)) + override = (options.source_override or '').strip().lower() + + if override: + source_chain = [override] + [source for source in source_chain if source != override] + + if not options.allow_fallback: + source_chain = source_chain[:1] + + return source_chain + + +def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any: + if value is None: + return default + + for name in names: + if isinstance(value, dict): + if name in value and value[name] is not None: + return value[name] + else: + candidate = getattr(value, name, None) + if candidate is not None: + return candidate + return default + + def get_client_for_source(source: str): """Get the client object for an exact metadata source. @@ -235,35 +264,6 @@ def get_artist_albums_for_source( return None -def _get_source_chain_for_lookup(options: MetadataLookupOptions) -> List[str]: - primary_source = get_primary_source() - source_chain = list(get_source_priority(primary_source)) - override = (options.source_override or '').strip().lower() - - if override: - source_chain = [override] + [source for source in source_chain if source != override] - - if not options.allow_fallback: - source_chain = source_chain[:1] - - return source_chain - - -def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any: - if value is None: - return default - - for name in names: - if isinstance(value, dict): - if name in value and value[name] is not None: - return value[name] - else: - candidate = getattr(value, name, None) - if candidate is not None: - return candidate - return default - - def _normalize_artist_name(value: Any) -> str: return (value or '').strip().casefold() @@ -841,234 +841,6 @@ def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: return normalized -def _build_single_import_context_payload( - track_data: Any, - source: Optional[str], - source_priority: List[str], - requested_title: str = '', - requested_artist: str = '', -) -> Dict[str, Any]: - album_data = _extract_lookup_value(track_data, 'album', default=None) - - track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '') - track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track' - track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[])) - if not track_artists and requested_artist: - track_artists = [{'name': requested_artist}] - - primary_track_artist = track_artists[0] if track_artists else {} - primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist' - primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '') - - album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or '' - album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '') - release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '') - album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album') - total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0) - album_images: List[Dict[str, Any]] = [] - album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '') - album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[])) - - if isinstance(album_data, dict): - album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name - album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) - release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) - album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) - total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) - album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] - if not album_image_url: - album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') - if not album_image_url and album_images: - album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') - album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) - elif album_data: - album_name = album_name or str(album_data) - - if not album_artists and primary_artist_name: - album_artists = [{'name': primary_artist_name}] - - if not album_image_url and album_images: - album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') - - track_info = { - 'id': track_id, - 'name': track_name, - 'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1), - 'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1), - 'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0), - 'artists': track_artists or [{'name': primary_artist_name}], - 'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''), - 'album': album_name, - 'album_id': album_id, - 'album_type': album_type, - 'release_date': release_date, - '_source': source or '', - } - - album_payload = { - 'id': album_id, - 'name': album_name, - 'release_date': release_date, - 'total_tracks': total_tracks or 1, - 'album_type': album_type, - 'image_url': album_image_url, - 'images': album_images, - 'artists': album_artists, - '_source': source or '', - } - - artist_payload = { - 'id': primary_artist_id, - 'name': primary_artist_name, - 'genres': [], - '_source': source or '', - } - - original_search = { - 'title': track_name, - 'artist': primary_artist_name, - 'album': album_name, - 'track_number': track_info['track_number'], - 'disc_number': track_info['disc_number'], - 'clean_title': track_name, - 'clean_album': album_name, - 'clean_artist': primary_artist_name, - 'artists': track_info['artists'], - 'duration_ms': track_info['duration_ms'], - 'id': track_id, - '_source': source or '', - } - - return { - 'success': bool(track_id or track_name != requested_title or album_name), - 'source': source, - 'source_priority': source_priority, - 'context': { - 'artist': artist_payload, - 'album': album_payload, - 'track_info': track_info, - 'original_search_result': original_search, - 'is_album_download': False, - 'has_clean_metadata': bool(track_id), - 'has_full_metadata': bool(track_id), - 'source': source, - 'source_priority': source_priority, - }, - } - - -def _build_single_import_fallback_context( - requested_title: str, - requested_artist: str, - source_priority: List[str], -) -> Dict[str, Any]: - artist_name = requested_artist or 'Unknown Artist' - title = requested_title or 'Unknown Track' - return { - 'success': False, - 'source': None, - 'source_priority': source_priority, - 'context': { - 'artist': { - 'id': '', - 'name': artist_name, - 'genres': [], - '_source': '', - }, - 'album': { - 'id': '', - 'name': '', - 'release_date': '', - 'total_tracks': 1, - 'album_type': 'album', - 'image_url': '', - 'images': [], - 'artists': [], - '_source': '', - }, - 'track_info': { - 'id': '', - 'name': title, - 'track_number': 1, - 'disc_number': 1, - 'duration_ms': 0, - 'artists': [{'name': artist_name}], - 'uri': '', - 'album': '', - 'album_id': '', - 'album_type': 'album', - 'release_date': '', - '_source': '', - }, - 'original_search_result': { - 'title': title, - 'artist': artist_name, - 'album': '', - 'track_number': 1, - 'disc_number': 1, - 'clean_title': title, - 'clean_album': '', - 'clean_artist': artist_name, - 'artists': [{'name': artist_name}], - 'duration_ms': 0, - 'id': '', - '_source': '', - }, - 'is_album_download': False, - 'has_clean_metadata': False, - 'has_full_metadata': False, - 'source': None, - 'source_priority': source_priority, - }, - } - - -def _build_track_search_query(source: str, title: str, artist: str) -> str: - base_query = " ".join(part for part in (title, artist) if part).strip() - if source == 'deezer' and title: - if artist: - return f'artist:"{artist}" track:"{title}"' - return f'track:"{title}"' - return base_query or title or artist - - -def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]: - if not search_results: - return None - - target_title = str(title or '').strip().lower() - target_artist = str(artist or '').strip().lower() - - for candidate in search_results: - candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower() - if candidate_title != target_title: - continue - - if not target_artist: - return candidate - - candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[])) - candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else '' - if candidate_artist_name == target_artist: - return candidate - - return search_results[0] - - -def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: - if not client or not hasattr(client, 'search_tracks'): - return [] - - try: - kwargs = {'limit': limit} - if source == 'spotify': - kwargs['allow_fallback'] = False - return client.search_tracks(query, **kwargs) or [] - except Exception as exc: - logger.debug("Could not search %s for %s: %s", source, query, exc) - return [] - - def get_single_track_import_context( title: str, artist: str = '', @@ -1076,94 +848,16 @@ def get_single_track_import_context( override_source: str = 'spotify', source_override: Optional[str] = None, ) -> Dict[str, Any]: - """Build an import context for singles using source-priority metadata lookup.""" - options = MetadataLookupOptions(source_override=source_override, allow_fallback=True) - source_priority = _get_source_chain_for_lookup(options) - title = (title or '').strip() - artist = (artist or '').strip() + """Compatibility wrapper for the single-track import resolver.""" + from core.import_resolution import get_single_track_import_context as _get_single_track_import_context - if override_id: - chosen_source = (override_source or 'spotify').strip().lower() or 'spotify' - client = get_client_for_source(chosen_source) - if client and hasattr(client, 'get_track_details'): - try: - track_data = client.get_track_details(str(override_id)) - if track_data: - payload = _build_single_import_context_payload( - track_data, - chosen_source, - source_priority, - requested_title=title, - requested_artist=artist, - ) - if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): - try: - artist_details = client.get_artist(payload['context']['artist']['id']) - if artist_details: - payload['context']['artist']['genres'] = _extract_lookup_value( - artist_details, - 'genres', - default=[], - ) or [] - except Exception: - pass - return payload - except Exception as exc: - logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc) - - for source in source_priority: - client = get_client_for_source(source) - if not client: - continue - - search_query = _build_track_search_query(source, title, artist) - if not search_query: - continue - - search_results = _search_tracks_for_source(source, client, search_query, limit=5) - if not search_results and search_query != title: - search_results = _search_tracks_for_source(source, client, title, limit=5) - if not search_results and artist and search_query != artist: - search_results = _search_tracks_for_source(source, client, artist, limit=5) - - if not search_results: - continue - - best_match = _pick_best_track_match(search_results, title or search_query, artist) - if not best_match: - continue - - resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '') - resolved_data = best_match - if resolved_track_id and hasattr(client, 'get_track_details'): - try: - detailed = client.get_track_details(resolved_track_id) - if detailed: - resolved_data = detailed - except Exception as exc: - logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc) - - payload = _build_single_import_context_payload( - resolved_data, - source, - source_priority, - requested_title=title, - requested_artist=artist, - ) - if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): - try: - artist_details = client.get_artist(payload['context']['artist']['id']) - if artist_details: - payload['context']['artist']['genres'] = _extract_lookup_value( - artist_details, - 'genres', - default=[], - ) or [] - except Exception: - pass - return payload - - return _build_single_import_fallback_context(title, artist, source_priority) + return _get_single_track_import_context( + title, + artist=artist, + override_id=override_id, + override_source=override_source, + source_override=source_override, + ) def resolve_album_reference( diff --git a/core/metadata_source.py b/core/metadata_source.py new file mode 100644 index 00000000..ce1ddbe4 --- /dev/null +++ b/core/metadata_source.py @@ -0,0 +1,896 @@ +"""Source metadata extraction and source-ID embedding helpers.""" + +from __future__ import annotations + +import re +import threading +import time +from typing import Any, Dict + +from core.import_context import ( + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + get_source_tag_names, + normalize_import_context, +) +from core.metadata_common import ( + _extract_artist_name, + _get_config_manager, + _get_database, + _get_itunes_client, + _get_logger, + _get_mutagen_symbols, + _is_vorbis_like, +) + + +_MB_RELEASE_CACHE: Dict[tuple, str] = {} +_MB_RELEASE_CACHE_LOCK = threading.RLock() +_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} +_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() + +_EDITION_PAREN_RE = re.compile( + r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]', + re.IGNORECASE, +) +_EDITION_BARE_RE = re.compile( + r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?\s*$', + re.IGNORECASE, +) + + +def _normalize_album_cache_key(album_name: str) -> str: + result = _EDITION_PAREN_RE.sub("", album_name or "") + result = _EDITION_BARE_RE.sub("", result) + return result.lower().strip() + + +def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> dict: + if album_info is None: + album_info = {} + + cfg = _get_config_manager() + logger_ = _get_logger() + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + + artist_dict = artist if isinstance(artist, dict) else { + "name": _extract_artist_name(artist), + "id": getattr(artist, "id", ""), + "genres": list(getattr(artist, "genres", []) or []), + } + + metadata: Dict[str, Any] = { + "source": source, + "source_track_id": source_ids["track_id"], + "source_artist_id": source_ids["artist_id"], + "source_album_id": source_ids["album_id"], + } + + metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "")) + if original_search.get("clean_title"): + logger_.info("Metadata: Using clean title: '%s'", metadata["title"]) + elif album_info.get("clean_track_name"): + logger_.info("Metadata: Using album info clean name: '%s'", metadata["title"]) + else: + logger_.warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) + + artists = original_search.get("artists") + if isinstance(artists, list) and artists: + all_artists = [] + for artist_item in artists: + if isinstance(artist_item, dict) and artist_item.get("name"): + all_artists.append(artist_item["name"]) + elif isinstance(artist_item, str): + all_artists.append(artist_item) + else: + all_artists.append(str(artist_item)) + metadata["artist"] = ", ".join(all_artists) + logger_.info("Metadata: Using all artists: '%s'", metadata["artist"]) + else: + metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context) + logger_.info("Metadata: Using primary artist: '%s'", metadata["artist"]) + + raw_album_artist = artist_dict.get("name", "") or metadata["artist"] + track_info_ctx = track_info or {} + explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None + album_artists_for_collab = None + + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + raw_album_artist = explicit_artist["name"] + album_artists_for_collab = [explicit_artist] + elif isinstance(explicit_artist, str) and explicit_artist: + raw_album_artist = explicit_artist + album_artists_for_collab = [{"name": explicit_artist}] + elif album_ctx and isinstance(album_ctx, dict): + album_artists = album_ctx.get("artists", []) + if album_artists: + first_album_artist = album_artists[0] + if isinstance(first_album_artist, dict) and first_album_artist.get("name"): + raw_album_artist = first_album_artist["name"] + elif isinstance(first_album_artist, str) and first_album_artist: + raw_album_artist = first_album_artist + album_artists_for_collab = album_artists + + collab_mode = cfg.get("file_organization.collab_artist_mode", "first") + if collab_mode == "first" and raw_album_artist: + context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or [] + if len(context_artists) > 1: + first = context_artists[0] + raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first) + elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist): + artist_id = str(artist_dict.get("id", "")) + if source == "itunes" and artist_id.isdigit(): + try: + itunes_client = _get_itunes_client() + if itunes_client and hasattr(itunes_client, "resolve_primary_artist"): + resolved = itunes_client.resolve_primary_artist(artist_id) + if resolved and resolved != raw_album_artist: + raw_album_artist = resolved + except Exception: + pass + metadata["album_artist"] = raw_album_artist + + if album_info.get("is_album"): + metadata["album"] = album_info.get("album_name", "Unknown Album") + metadata["track_number"] = album_info.get("track_number", 1) + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1 + logger_.info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) + else: + if album_ctx and album_ctx.get("name"): + logger_.info("[SAFEGUARD] Using album context name instead of track title for album metadata") + metadata["album"] = album_ctx["name"] + metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1 + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) + else: + metadata["album"] = metadata["title"] + metadata["track_number"] = 1 + metadata["total_tracks"] = 1 + + disc_num = original_search.get("disc_number") + if disc_num is None and album_info: + disc_num = album_info.get("disc_number") + metadata["disc_number"] = disc_num if disc_num is not None else 1 + + if album_ctx and album_ctx.get("release_date"): + metadata["date"] = album_ctx["release_date"][:4] + + genres = artist_dict.get("genres") or [] + if genres: + from core.genre_filter import filter_genres + + filtered = filter_genres(list(genres[:2]), cfg) + if filtered: + metadata["genre"] = ", ".join(filtered) + + metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None + if not metadata["album_art_url"] and album_ctx: + album_image = album_ctx.get("image_url") + if not album_image and album_ctx.get("images"): + first_image = album_ctx["images"][0] + album_image = first_image.get("url") if isinstance(first_image, dict) else None + metadata["album_art_url"] = album_image + + logger_.info( + "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", + metadata.get("title"), + metadata.get("artist"), + metadata.get("album_artist"), + metadata.get("album"), + metadata.get("track_number"), + metadata.get("total_tracks"), + metadata.get("disc_number"), + ) + + return metadata + + +def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): + cfg = _get_config_manager() + logger_ = _get_logger() + symbols = _get_mutagen_symbols() + if not symbols: + return + + try: + tag_config = { + "SPOTIFY_TRACK_ID": "spotify.tags.track_id", + "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", + "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", + "ITUNES_TRACK_ID": "itunes.tags.track_id", + "ITUNES_ARTIST_ID": "itunes.tags.artist_id", + "ITUNES_ALBUM_ID": "itunes.tags.album_id", + "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", + "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", + "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", + "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", + "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", + "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", + "RELEASETYPE": "musicbrainz.tags.release_type", + "ORIGINALDATE": "musicbrainz.tags.original_date", + "RELEASESTATUS": "musicbrainz.tags.release_status", + "RELEASECOUNTRY": "musicbrainz.tags.release_country", + "BARCODE": "musicbrainz.tags.barcode", + "MEDIA": "musicbrainz.tags.media", + "TOTALDISCS": "musicbrainz.tags.total_discs", + "CATALOGNUMBER": "musicbrainz.tags.catalog_number", + "SCRIPT": "musicbrainz.tags.script", + "ASIN": "musicbrainz.tags.asin", + "DEEZER_TRACK_ID": "deezer.tags.track_id", + "DEEZER_ARTIST_ID": "deezer.tags.artist_id", + "AUDIODB_TRACK_ID": "audiodb.tags.track_id", + "TIDAL_TRACK_ID": "tidal.tags.track_id", + "TIDAL_ARTIST_ID": "tidal.tags.artist_id", + "QOBUZ_TRACK_ID": "qobuz.tags.track_id", + "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", + "GENIUS_TRACK_ID": "genius.tags.track_id", + } + + def _tag_enabled(path: str) -> bool: + return cfg.get(path, True) is not False + + def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: + if not a or not b: + return False + from difflib import SequenceMatcher + + norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() + return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold + + context = normalize_import_context(context) + source = (metadata.get("source") or "").strip().lower() + source_ids = {} + if source: + source_tag_names = get_source_tag_names(source) + source_track_id = metadata.get("source_track_id") + source_artist_id = metadata.get("source_artist_id") + source_album_id = metadata.get("source_album_id") + if cfg.get(f"{source}.embed_tags", True) is not False: + if source_tag_names.get("track") and source_track_id: + source_ids[source_tag_names["track"]] = source_track_id + if source_tag_names.get("artist") and source_artist_id: + source_ids[source_tag_names["artist"]] = source_artist_id + if source_tag_names.get("album") and source_album_id: + source_ids[source_tag_names["album"]] = source_album_id + + if not source_ids: + if cfg.get("spotify.embed_tags", True) is not False: + if metadata.get("spotify_track_id"): + source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] + if metadata.get("spotify_artist_id"): + source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] + if metadata.get("spotify_album_id"): + source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] + if cfg.get("itunes.embed_tags", True) is not False: + if metadata.get("itunes_track_id"): + source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] + if metadata.get("itunes_artist_id"): + source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] + if metadata.get("itunes_album_id"): + source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] + + track_title = metadata.get("title", "") + artist_name = metadata.get("album_artist", "") or metadata.get("artist", "") + track_info = get_import_track_info(context) + explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None + batch_artist_name = None + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + batch_artist_name = explicit_artist["name"] + elif isinstance(explicit_artist, str) and explicit_artist: + batch_artist_name = explicit_artist + + pp = { + "id_tags": source_ids, + "track_title": track_title, + "artist_name": artist_name, + "batch_artist_name": batch_artist_name, + "metadata": metadata, + "recording_mbid": None, + "artist_mbid": None, + "release_mbid": "", + "mb_genres": [], + "isrc": None, + "deezer_bpm": None, + "deezer_isrc": None, + "audiodb_mood": None, + "audiodb_style": None, + "audiodb_genre": None, + "tidal_isrc": None, + "tidal_copyright": None, + "qobuz_isrc": None, + "qobuz_copyright": None, + "qobuz_label": None, + "lastfm_tags": [], + "lastfm_url": None, + "genius_url": None, + "release_year": None, + } + + source_order = cfg.get("metadata_enhancement.post_process_order", None) + if not isinstance(source_order, list) or not source_order: + source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] + + db = _get_database() + + for source_name in source_order: + if source_name == "musicbrainz": + if cfg.get("musicbrainz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + mb_worker = getattr(runtime, "mb_worker", None) + mb_service = mb_worker.mb_service if mb_worker else None + if not mb_service: + continue + try: + result = mb_service.match_recording(track_title, artist_name) + if result and result.get("mbid"): + pp["recording_mbid"] = result["mbid"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] + details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) + if details: + isrcs = details.get("isrcs", []) + if isrcs: + pp["isrc"] = isrcs[0] + pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] + + track_artist_name = metadata.get("artist", "") or artist_name + if ", " in track_artist_name: + track_artist_name = track_artist_name.split(", ")[0] + artist_result = mb_service.match_artist(track_artist_name) + if artist_result and artist_result.get("mbid"): + pp["artist_mbid"] = artist_result["mbid"] + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] + + album_name_for_mb = metadata.get("album", "") + if album_name_for_mb: + artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() + rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) + rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) + with _MB_RELEASE_CACHE_LOCK: + cached = _MB_RELEASE_CACHE.get(rc_key_norm) + if cached is None: + cached = _MB_RELEASE_CACHE.get(rc_key_exact) + if cached is not None: + pp["release_mbid"] = cached + else: + try: + rc_result = mb_service.match_release(album_name_for_mb, artist_name) + pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" + except Exception: + pp["release_mbid"] = "" + _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] + _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] + if pp["release_mbid"]: + pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] + + if pp["release_mbid"]: + with _MB_RELEASE_DETAIL_CACHE_LOCK: + release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) + if release_detail is None: + release_detail = mb_service.mb_client.get_release( + pp["release_mbid"], + includes=["release-groups", "labels", "media", "artist-credits", "recordings"], + ) or {} + with _MB_RELEASE_DETAIL_CACHE_LOCK: + _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail + if release_detail: + rg = release_detail.get("release-group", {}) + if rg.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] + ac = release_detail.get("artist-credit", []) + if ac and isinstance(ac[0], dict): + aa = ac[0].get("artist", {}) + if aa.get("id"): + pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] + if rg.get("primary-type"): + pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] + if rg.get("first-release-date"): + pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] + if not pp["release_year"] and len(rg["first-release-date"]) >= 4: + year = rg["first-release-date"][:4] + if year.isdigit(): + pp["release_year"] = year + if release_detail.get("status"): + pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] + if release_detail.get("country"): + pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] + if release_detail.get("barcode"): + pp["id_tags"]["BARCODE"] = release_detail["barcode"] + media_list = release_detail.get("media", []) + if media_list: + fmt = media_list[0].get("format", "") + if fmt: + pp["id_tags"]["MEDIA"] = fmt + pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) + label_info = release_detail.get("label-info", []) + if label_info and isinstance(label_info[0], dict): + cat = label_info[0].get("catalog-number", "") + if cat: + pp["id_tags"]["CATALOGNUMBER"] = cat + text_rep = release_detail.get("text-representation", {}) + if isinstance(text_rep, dict) and text_rep.get("script"): + pp["id_tags"]["SCRIPT"] = text_rep["script"] + if release_detail.get("asin"): + pp["id_tags"]["ASIN"] = release_detail["asin"] + track_num = metadata.get("track_number") + disc_num = metadata.get("disc_number") or 1 + if track_num and media_list: + try: + track_num_int = int(track_num) + disc_num_int = int(disc_num) + for medium in media_list: + if medium.get("position", 1) == disc_num_int: + for mtrack in (medium.get("tracks") or medium.get("track-list", [])): + if mtrack.get("position") == track_num_int: + if mtrack.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] + release_recording = mtrack.get("recording", {}) + if release_recording.get("id"): + pp["recording_mbid"] = release_recording["id"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] + break + break + except (ValueError, TypeError): + pass + except Exception as exc: + logger_.error("MusicBrainz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "deezer": + if cfg.get("deezer.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + deezer_worker = getattr(runtime, "deezer_worker", None) + dz_client = deezer_worker.client if deezer_worker else None + if not dz_client: + continue + dz_result = dz_client.search_track(artist_name, track_title) + if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): + dz_track_id = dz_result["id"] + pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) + dz_artist_id = dz_result.get("artist", {}).get("id") + if dz_artist_id: + pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) + dz_details = dz_client.get_track_details(dz_track_id) + if dz_details: + bpm_val = dz_details.get("bpm") + if bpm_val and bpm_val > 0: + pp["deezer_bpm"] = bpm_val + dz_isrc = dz_details.get("isrc") + if dz_isrc: + pp["deezer_isrc"] = dz_isrc + if not pp["release_year"]: + dz_album = dz_result.get("album", {}) + dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" + if len(dz_release) >= 4 and dz_release[:4].isdigit(): + pp["release_year"] = dz_release[:4] + except Exception as exc: + logger_.error("Deezer lookup failed (non-fatal): %s", exc) + continue + + if source_name == "audiodb": + if cfg.get("audiodb.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + audiodb_worker = getattr(runtime, "audiodb_worker", None) + adb_client = audiodb_worker.client if audiodb_worker else None + if not adb_client: + continue + adb_result = adb_client.search_track(artist_name, track_title) + if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): + adb_track_id = adb_result.get("idTrack") + if adb_track_id: + pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) + adb_mb_track = adb_result.get("strMusicBrainzID") + if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track + pp["recording_mbid"] = adb_mb_track + adb_mb_artist = adb_result.get("strMusicBrainzArtistID") + if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist + pp["artist_mbid"] = adb_mb_artist + pp["audiodb_mood"] = adb_result.get("strMood") or None + pp["audiodb_style"] = adb_result.get("strStyle") or None + pp["audiodb_genre"] = adb_result.get("strGenre") or None + except Exception as exc: + logger_.error("AudioDB lookup failed (non-fatal): %s", exc) + continue + + if source_name == "tidal": + if cfg.get("tidal.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + tidal_client = getattr(runtime, "tidal_client", None) + if not (tidal_client and tidal_client.is_authenticated()): + continue + td_result = tidal_client.search_track(artist_name, track_title) + if td_result and _names_match(td_result.get("title", ""), track_title): + td_track_id = td_result.get("id") + if td_track_id: + pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) + td_artist = td_result.get("artist", {}) + if isinstance(td_artist, dict) and td_artist.get("id"): + pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) + if td_track_id: + td_details = tidal_client.get_track(str(td_track_id)) + if td_details: + pp["tidal_isrc"] = td_details.get("isrc") + td_copyright = td_details.get("copyright") + if isinstance(td_copyright, dict): + td_copyright = td_copyright.get("text", td_copyright.get("name", "")) + pp["tidal_copyright"] = td_copyright or None + if not pp["release_year"]: + td_album = td_result.get("album", {}) + td_release = "" + if isinstance(td_album, dict): + td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") + if len(td_release) >= 4 and td_release[:4].isdigit(): + pp["release_year"] = td_release[:4] + except Exception as exc: + logger_.error("Tidal lookup failed (non-fatal): %s", exc) + continue + + if source_name == "qobuz": + if cfg.get("qobuz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) + qz_client = qobuz_worker.client if qobuz_worker else None + if not (qz_client and qz_client.is_authenticated()): + continue + qz_result = qz_client.search_track(artist_name, track_title) + if qz_result: + qz_performer = qz_result.get("performer") or {} + if not isinstance(qz_performer, dict): + qz_performer = {} + qz_artist_name = qz_performer.get("name", "") + if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): + qz_track_id = qz_result.get("id") + if qz_track_id: + pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) + if qz_performer.get("id"): + pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) + qz_isrc = qz_result.get("isrc") + if isinstance(qz_isrc, dict): + qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) + if qz_isrc: + pp["qobuz_isrc"] = qz_isrc + qz_copyright = qz_result.get("copyright") + if isinstance(qz_copyright, dict): + qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) + if isinstance(qz_copyright, str): + pp["qobuz_copyright"] = qz_copyright + qz_album = qz_result.get("album", {}) + if isinstance(qz_album, dict): + qz_label_info = qz_album.get("label", {}) + if isinstance(qz_label_info, dict) and qz_label_info.get("name"): + pp["qobuz_label"] = qz_label_info["name"] + if not pp["release_year"]: + qz_release = str(qz_album.get("release_date_original", "") or "") + if not qz_release: + qz_ts = qz_album.get("released_at") + if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: + import datetime as _dt + qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) + if len(qz_release) >= 4 and qz_release[:4].isdigit(): + pp["release_year"] = qz_release[:4] + except Exception as exc: + logger_.error("Qobuz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "lastfm": + if cfg.get("lastfm.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + lastfm_worker = getattr(runtime, "lastfm_worker", None) + lf_client = lastfm_worker.client if lastfm_worker else None + if not lf_client: + continue + lf_result = lf_client.get_track_info(artist_name, track_title) + if lf_result: + lf_url = lf_result.get("url") + if lf_url: + pp["lastfm_url"] = lf_url + lf_toptags = lf_result.get("toptags", {}) + if isinstance(lf_toptags, dict): + tag_list = lf_toptags.get("tag", []) + if isinstance(tag_list, list): + pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] + elif isinstance(tag_list, dict) and tag_list.get("name"): + pp["lastfm_tags"] = [tag_list["name"]] + except Exception as exc: + logger_.error("Last.fm lookup failed (non-fatal): %s", exc) + continue + + if source_name == "genius": + if cfg.get("genius.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + import core.genius_client as _genius_module + + if time.time() < _genius_module._rate_limit_until: + logger_.info("Genius rate-limited, skipping (non-blocking)") + continue + genius_worker = getattr(runtime, "genius_worker", None) + g_client = genius_worker.client if genius_worker else None + if not g_client: + continue + g_result = g_client.search_song(artist_name, track_title) + if g_result: + g_id = g_result.get("id") + if g_id: + pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) + g_url = g_result.get("url") + if g_url: + pp["genius_url"] = g_url + except Exception as exc: + logger_.error("Genius lookup failed (non-fatal): %s", exc) + continue + + if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: + return + + filtered_tags: Dict[str, str] = {} + for tag_name, value in pp["id_tags"].items(): + config_path = tag_config.get(tag_name) + if config_path and not _tag_enabled(config_path): + continue + filtered_tags[tag_name] = value + + written = [] + id3_tag_map = { + "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), + "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), + "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), + "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), + "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), + "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), + "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), + "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), + "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), + "ORIGINALDATE": ("TDOR", None), + "MEDIA": ("TMED", None), + } + vorbis_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", + "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", + "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", + "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", + "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", + "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", + } + mp4_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", + "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", + "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", + "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", + "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", + "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", + "RELEASETYPE": "MusicBrainz Album Type", + "RELEASESTATUS": "MusicBrainz Album Status", + "RELEASECOUNTRY": "MusicBrainz Album Release Country", + } + + if isinstance(audio_file.tags, symbols.ID3): + for tag_name, value in filtered_tags.items(): + spec = id3_tag_map.get(tag_name) + if spec: + frame_type, desc = spec + if frame_type == "UFID": + audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) + written.append(f"UFID:{desc}") + elif frame_type == "TDOR": + audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) + written.append("TDOR") + elif frame_type == "TMED": + audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) + written.append("TMED") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) + written.append(f"TXXX:{desc}") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) + written.append(f"TXXX:{tag_name}") + elif isinstance(audio_file, symbols.MP4): + # Keep the dedicated MP4 path last so the same tag maps can be reused. + for tag_name, value in filtered_tags.items(): + key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" + audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] + written.append(key) + elif _is_vorbis_like(audio_file, symbols): + for tag_name, value in filtered_tags.items(): + audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] + written.append(vorbis_tag_map.get(tag_name, tag_name)) + + if written: + logger_.info("Embedded IDs: %s", ", ".join(written)) + + release_year = pp["release_year"] + needs_date_tag = bool(release_year and not metadata.get("date")) + if needs_date_tag: + metadata["date"] = release_year + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["date"] = [release_year] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9day"] = [release_year] + logger_.info("Date tag: %s", release_year) + + if _tag_enabled("deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: + bpm_int = int(pp["deezer_bpm"]) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["BPM"] = [str(bpm_int)] + elif isinstance(audio_file, symbols.MP4): + audio_file["tmpo"] = [bpm_int] + logger_.info("BPM: %s", bpm_int) + + if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["MOOD"] = [pp["audiodb_mood"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] + + if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["STYLE"] = [pp["audiodb_style"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] + + if _tag_enabled("metadata_enhancement.tags.genre_merge"): + enrichment_genres = [] + if _tag_enabled("musicbrainz.tags.genres"): + enrichment_genres += pp["mb_genres"] + if pp["audiodb_genre"] and _tag_enabled("audiodb.tags.genre"): + enrichment_genres.append(pp["audiodb_genre"]) + if _tag_enabled("lastfm.tags.genres"): + enrichment_genres += pp["lastfm_tags"] + if enrichment_genres: + from core.genre_filter import filter_genres as _filter_genres + + enrichment_genres = _filter_genres(enrichment_genres, cfg) + source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] + seen = set() + merged = [] + for genre in source_genres + enrichment_genres: + key = genre.strip().lower() + if key and key not in seen: + seen.add(key) + merged.append(genre.strip().title()) + if len(merged) >= 5: + break + if merged: + genre_string = ", ".join(merged) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENRE"] = [genre_string] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9gen"] = [genre_string] + logger_.info("Genres merged: %s", genre_string) + + isrc_candidates = [] + if pp["isrc"] and _tag_enabled("musicbrainz.tags.isrc"): + isrc_candidates.append(("MusicBrainz", pp["isrc"])) + if pp["deezer_isrc"] and _tag_enabled("deezer.tags.isrc"): + isrc_candidates.append(("Deezer", pp["deezer_isrc"])) + if pp["tidal_isrc"] and _tag_enabled("tidal.tags.isrc"): + isrc_candidates.append(("Tidal", pp["tidal_isrc"])) + if pp["qobuz_isrc"] and _tag_enabled("qobuz.tags.isrc"): + isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) + if isrc_candidates: + isrc_source, final_isrc = isrc_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["ISRC"] = [final_isrc] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] + logger_.info("ISRC (%s): %s", isrc_source, final_isrc) + + copyright_candidates = [] + if pp["tidal_copyright"] and _tag_enabled("tidal.tags.copyright"): + copyright_candidates.append(("Tidal", pp["tidal_copyright"])) + if pp["qobuz_copyright"] and _tag_enabled("qobuz.tags.copyright"): + copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) + if copyright_candidates: + copyright_source, final_copyright = copyright_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["COPYRIGHT"] = [final_copyright] + elif isinstance(audio_file, symbols.MP4): + audio_file["cprt"] = [final_copyright] + logger_.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) + + if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LABEL"] = [pp["qobuz_label"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] + + if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LASTFM_URL"] = [pp["lastfm_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] + + if _tag_enabled("genius.tags.url") and pp["genius_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENIUS_URL"] = [pp["genius_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] + + release_id = pp["release_mbid"] + if release_id: + metadata["musicbrainz_release_id"] = release_id + if db is not None: + try: + album_name_for_db = metadata.get("album", "") + album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") + if album_name_for_db and album_artist_for_db: + conn = db._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE albums SET year = ? + WHERE (year IS NULL OR year = 0) + AND id IN ( + SELECT al.id FROM albums al + JOIN artists ar ON ar.id = al.artist_id + WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) + ) + """, + (int(release_year), album_name_for_db, album_artist_for_db), + ) + if cursor.rowcount > 0: + conn.commit() + logger_.info("Updated album year to %s in database", release_year) + else: + conn.rollback() + finally: + conn.close() + except Exception as exc: + logger_.error("Could not update album year in DB: %s", exc) + + except Exception as exc: + logger_.error("Error embedding source IDs (non-fatal): %s", exc) diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py index 49cbe630..8f557972 100644 --- a/tests/test_metadata_enrichment.py +++ b/tests/test_metadata_enrichment.py @@ -1,9 +1,8 @@ -import logging import types -import pytest - from core import metadata_enrichment as me +from core import metadata_artwork as ma +from core import metadata_source as ms class _Config: @@ -110,20 +109,9 @@ def _fake_symbols(audio): ) -def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(): - class _ItunesClient: - def __init__(self): - self.called = False - - def resolve_primary_artist(self, artist_id): - self.called = True - raise AssertionError("itunes fallback should not run for non-itunes sources") - - runtime = types.SimpleNamespace( - logger=logging.getLogger("test.metadata_enrichment"), - config_manager=_Config({"file_organization.collab_artist_mode": "first"}), - itunes_enrichment_worker=types.SimpleNamespace(client=_ItunesClient()), - ) +def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(monkeypatch): + monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config({"file_organization.collab_artist_mode": "first"})) + monkeypatch.setattr(ms, "_get_itunes_client", lambda: (_ for _ in ()).throw(AssertionError("itunes fallback should not run for non-itunes sources"))) context = { "source": "spotify", @@ -156,7 +144,6 @@ def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_ context, context["artist"], {"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 2, "album_image_url": "https://img.example/album.jpg"}, - runtime=runtime, ) assert metadata["source"] == "spotify" @@ -168,27 +155,14 @@ def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_ assert metadata["total_tracks"] == 12 assert metadata["disc_number"] == 2 assert metadata["album_art_url"] == "https://img.example/album.jpg" - assert runtime.itunes_enrichment_worker.client.called is False def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch): - runtime = types.SimpleNamespace( - logger=logging.getLogger("test.metadata_enrichment"), - config_manager=_Config(), - mb_worker=None, - deezer_worker=None, - audiodb_worker=None, - tidal_client=None, - qobuz_enrichment_worker=None, - lastfm_worker=None, - genius_worker=None, - itunes_enrichment_worker=None, - get_database=lambda: None, - ) - audio = _FakeAudio() symbols = _fake_symbols(audio) - monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config()) + monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "_get_database", lambda: None) current_metadata = { "source": "deezer", @@ -200,7 +174,7 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc "album_artist": "Artist One", "album": "Album One", } - me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}}) current_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] assert "DEEZER_TRACK_ID" in current_descs @@ -208,7 +182,7 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc audio = _FakeAudio() symbols = _fake_symbols(audio) - monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) legacy_metadata = { "source": "", @@ -223,7 +197,7 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc "album_artist": "Artist One", "album": "Album One", } - me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}}) legacy_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] assert "SPOTIFY_TRACK_ID" in legacy_descs @@ -237,35 +211,25 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch): audio = _FakeAudio() symbols = _fake_symbols(audio) - runtime = types.SimpleNamespace( - logger=logging.getLogger("test.metadata_enrichment"), - config_manager=_Config( - { - "metadata_enhancement.enabled": True, - "metadata_enhancement.embed_album_art": False, - "metadata_enhancement.tags.write_multi_artist": False, - } - ), - mb_worker=None, - deezer_worker=None, - audiodb_worker=None, - tidal_client=None, - qobuz_enrichment_worker=None, - lastfm_worker=None, - genius_worker=None, - itunes_enrichment_worker=None, - get_database=lambda: None, - ) - strip_calls = [] verify_calls = [] - monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) - monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path, runtime=None: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) + monkeypatch.setattr(me, "_get_config_manager", lambda: _Config( + { + "metadata_enhancement.enabled": True, + "metadata_enhancement.embed_album_art": False, + "metadata_enhancement.tags.write_multi_artist": False, + } + )) + monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config()) + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "_get_database", lambda: None) + monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) monkeypatch.setattr( me, "extract_source_metadata", - lambda context, artist, album_info, runtime=None: { + lambda context, artist, album_info: { "source": "deezer", "source_track_id": "dz-track", "source_artist_id": "dz-artist", @@ -283,7 +247,7 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch }, ) monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None) - monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path, runtime=None: verify_calls.append(file_path) or True) + monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path: verify_calls.append(file_path) or True) album_info = {} result = me.enhance_file_metadata( @@ -291,7 +255,6 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch {"_audio_quality": ""}, {"name": "Artist One"}, album_info, - runtime=runtime, ) assert result is True @@ -306,17 +269,14 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): - runtime = types.SimpleNamespace( - logger=logging.getLogger("test.metadata_enrichment"), - config_manager=_Config( - { - "metadata_enhancement.cover_art_download": True, - "metadata_enhancement.prefer_caa_art": False, - } - ), - ) + monkeypatch.setattr(ma, "_get_config_manager", lambda: _Config( + { + "metadata_enhancement.cover_art_download": True, + "metadata_enhancement.prefer_caa_art": False, + } + )) - monkeypatch.setattr(me.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes")) + monkeypatch.setattr(ma.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes")) target_dir = tmp_path / "Album One" target_dir.mkdir() @@ -325,7 +285,6 @@ def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): {}, str(target_dir), {"album": {"image_url": "https://img.example/album.jpg"}}, - runtime=runtime, ) cover_path = target_dir / "cover.jpg" diff --git a/web_server.py b/web_server.py index 504ba7b5..e6cd49c2 100644 --- a/web_server.py +++ b/web_server.py @@ -559,6 +559,8 @@ def _make_context_key(username, filename): normalized = filename.replace('\\', '/').lstrip('/') if filename else '' return f"{username}::{normalized}" +IS_SHUTTING_DOWN = False + # --- Initialize Core Application Components --- # Each client is initialized independently so one failure doesn't take down everything. # Previously, a single exception set ALL clients to None, breaking the entire app. @@ -650,6 +652,1563 @@ except Exception as e: logger.info("Core service initialization complete.") +# --- Shared Runtime State --- +# These globals are used by routes, background workers, and shutdown helpers. +# A prior refactor accidentally dropped this initializer block, so several +# modules and handlers were still referencing names that never got created. + +# Global Streaming State Management +stream_state = { + "status": "stopped", # States: stopped, loading, queued, ready, error + "progress": 0, + "track_info": None, + "file_path": None, # Path to the audio file in the Stream folder + "error_message": None, +} +stream_lock = threading.Lock() # Prevent race conditions +stream_background_task = None +stream_executor = ThreadPoolExecutor(max_workers=1) # Only one stream at a time + +# Global OAuth State Management +# Store PKCE values for Tidal OAuth flow +tidal_oauth_state = { + "code_verifier": None, + "code_challenge": None, +} +tidal_oauth_lock = threading.Lock() + +# Sync Page Globals +sync_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="SyncWorker") +active_sync_workers = {} # Key: playlist_id, Value: Future object +sync_states = {} # Key: playlist_id, Value: dict with progress info +sync_lock = threading.Lock() + +# Database Update / Tool Progress State +db_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DBUpdate") +db_update_worker = None +db_update_state = { + "status": "idle", # idle, running, finished, error + "phase": "Idle", + "progress": 0, + "current_item": "", + "processed": 0, + "total": 0, + "error_message": "", + "removed_artists": 0, + "removed_albums": 0, + "removed_tracks": 0, +} +_db_update_automation_id = None # Set when automation triggers DB update, used by callbacks +db_update_lock = threading.Lock() + +# Quality Scanner state +quality_scanner_state = { + "status": "idle", # idle, running, finished, error + "phase": "Ready to scan", + "progress": 0, + "processed": 0, + "total": 0, + "quality_met": 0, + "low_quality": 0, + "matched": 0, + "error_message": "", + "results": [], # List of low quality tracks with match status +} +quality_scanner_lock = threading.Lock() +quality_scanner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="QualityScanner") + +# Duplicate Cleaner state +duplicate_cleaner_state = { + "status": "idle", # idle, running, finished, error + "phase": "Ready to scan", + "progress": 0, + "files_scanned": 0, + "total_files": 0, + "duplicates_found": 0, + "deleted": 0, + "space_freed": 0, # in bytes + "error_message": "", +} +duplicate_cleaner_lock = threading.Lock() +duplicate_cleaner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DuplicateCleaner") + +# Retag Tool Globals +retag_state = { + "status": "idle", + "phase": "Ready", + "progress": 0, + "current_track": "", + "total_tracks": 0, + "processed": 0, + "error_message": "", +} +retag_lock = threading.Lock() +retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWorker") + +# Download Missing Tracks Modal State Management +# Thread-safe state tracking for modal download functionality. +# Shared task/batch state now lives in core.import_runtime_state. +missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") + +# Automatic Wishlist / Watchlist Processing Flags +# Processing state flags (guards/recovery - timers are now managed by AutomationEngine) +wishlist_auto_processing = False +wishlist_auto_processing_timestamp = 0 +wishlist_timer_lock = threading.Lock() + +watchlist_auto_scanning = False +watchlist_auto_scanning_timestamp = 0 +watchlist_timer_lock = threading.Lock() + +# Beatport Data Cache +# Cache Beatport scraping data to reduce load times and avoid hammering Beatport.com +beatport_data_cache = { + 'homepage': { + 'hero_tracks': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'top_10_lists': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'top_10_releases': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'new_releases': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'hype_picks': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'featured_charts': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'dj_charts': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + }, + 'genre': { + # Future expansion for genre-specific caching + # 'house': {'top_10': {...}, 'releases': {...}}, + # 'techno': {'top_10': {...}, 'releases': {...}} + }, + 'cache_lock': threading.Lock(), +} + +# Shared Soulseek transfer cache +# Keeps download status polling from hammering the API when multiple modals are open. +transfer_data_cache = { + 'data': {}, + 'last_update': 0, + 'update_lock': threading.Lock(), + 'cache_duration': 0.75, +} + + +def get_cached_transfer_data(): + """ + Return live Soulseek transfer data with a short cache window. + + The download modal, batch status endpoint, and socket emit loop all call + this helper, so we keep the API hit rate low while still refreshing often. + """ + current_time = time.time() + + with transfer_data_cache['update_lock']: + if (current_time - transfer_data_cache['last_update']) < transfer_data_cache['cache_duration']: + return transfer_data_cache['data'] + + if not soulseek_client: + return {} + + live_transfers_lookup = {} + try: + transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + if transfers_data: + for user_data in transfers_data: + if not isinstance(user_data, dict): + continue + username = user_data.get('username', 'Unknown') + for directory in user_data.get('directories', []) or []: + if not isinstance(directory, dict): + continue + for file_info in directory.get('files', []) or []: + if not isinstance(file_info, dict): + continue + transfer = dict(file_info) + transfer['username'] = username + lookup_key = _make_context_key(username, transfer.get('filename', '')) + live_transfers_lookup[lookup_key] = transfer + + transfer_data_cache['data'] = live_transfers_lookup + transfer_data_cache['last_update'] = current_time + except Exception as e: + logger.warning("Could not fetch live transfers (cached): %s", e) + return {} + + return live_transfers_lookup + + +def _regenerate_batch_m3u(batch, tracks): + """Regenerate an exported playlist M3U after downloads finish.""" + try: + from collections import defaultdict + from difflib import SequenceMatcher + + try: + from unidecode import unidecode as _unidecode + except ImportError: + _unidecode = lambda x: x + + from core.import_paths import sanitize_filename as _sanitize_filename + + def _norm(text): + return _unidecode(text).lower().strip() if text else '' + + def _track_name(track): + return track.get('name') or track.get('title') or 'Unknown' + + def _track_artist(track): + return track.get('artist') or track.get('artist_name') or 'Unknown' + + playlist_name = batch.get('playlist_name', 'Playlist') + db = get_database() + active_server = config_manager.get_active_media_server() + raw_base = config_manager.get('m3u_export.entry_base_path', '') or '' + entry_base_path = raw_base.rstrip('/\\') + + artist_groups = defaultdict(list) + for idx, track in enumerate(tracks): + artist_groups[_track_artist(track)].append((idx, track)) + + file_path_map = {} + for artist, group in artist_groups.items(): + if not artist or artist == 'Unknown': + for idx, _ in group: + file_path_map[idx] = None + continue + + try: + db_tracks = db.search_tracks(artist=artist, limit=500, server_source=active_server) + except Exception as search_err: + logger.debug("[M3U] Track lookup failed for %s: %s", artist, search_err) + db_tracks = [] + + if not db_tracks: + for idx, _ in group: + file_path_map[idx] = None + continue + + db_entries = [] + for db_track in db_tracks: + db_title = getattr(db_track, 'title', '') or '' + db_entries.append((_norm(db_title), db_track)) + + for idx, track in group: + name = _track_name(track) + if not name: + file_path_map[idx] = None + continue + + s_norm = _norm(name) + matched = None + for db_n, db_track in db_entries: + if not db_n: + continue + if s_norm == db_n or SequenceMatcher(None, s_norm, db_n).ratio() >= 0.7: + matched = db_track + break + + if matched is None: + file_path_map[idx] = None + else: + file_path = getattr(matched, 'file_path', None) + if file_path is None and isinstance(matched, dict): + file_path = matched.get('file_path') + file_path_map[idx] = file_path or None + + lines = ['#EXTM3U', f'#PLAYLIST:{playlist_name}', f'#GENERATED:{datetime.utcnow().isoformat()}Z', ''] + found = 0 + missing = 0 + for idx, track in enumerate(tracks): + duration_ms = track.get('duration_ms', 0) or 0 + artist = _track_artist(track) + name = _track_name(track) + lines.append(f'#EXTINF:{int(duration_ms / 1000)},{artist} - {name}') + file_path = file_path_map.get(idx) + if file_path: + path = f'{entry_base_path}/{file_path}' if entry_base_path else file_path + lines.append(path.replace('\\', '/')) + found += 1 + else: + lines.append(f'# MISSING: {artist} - {name}') + missing += 1 + lines.append('') + + if found == 0: + logger.info("[M3U] Skipping regeneration for %s: no library paths resolved", playlist_name) + return + + transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) + playlist_dir = _compute_m3u_folder(transfer_dir, 'playlist', playlist_name, '', '', '') + os.makedirs(playlist_dir, exist_ok=True) + m3u_path = os.path.join(playlist_dir, f'{_sanitize_filename(playlist_name)}.m3u') + + with open(m3u_path, 'w', encoding='utf-8') as f: + f.write('\n'.join(lines)) + + logger.info( + "[M3U] Regenerated M3U on batch complete: %s (%s/%s resolved, %s missing)", + m3u_path, + found, + len(tracks), + missing, + ) + except Exception as e: + logger.error("[M3U] Error in _regenerate_batch_m3u: %s", e) + + +def _sanitize_filename(filename: str) -> str: + """Sanitize a filename for filesystem use.""" + sanitized = re.sub(r'[<>:"/\\|?*]', '_', filename or '') + sanitized = re.sub(r'\s+', ' ', sanitized).strip() + sanitized = sanitized.rstrip('. ') or '_' + if re.match(r'^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)', sanitized, re.IGNORECASE): + sanitized = '_' + sanitized + return sanitized[:200] + + +def _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name='', album_name='', year=''): + """Compute the target folder for an M3U file using the template system.""" + from core.import_paths import get_file_path_from_template + + if context_type == 'album' and artist_name and album_name: + template_context = { + 'artist': artist_name, + 'albumartist': artist_name, + 'album': album_name, + 'title': 'placeholder', + 'track_number': 1, + 'disc_number': 1, + 'year': year, + 'quality': '' + } + folder_path, _ = get_file_path_from_template(template_context, 'album_path') + if folder_path: + return os.path.join(transfer_dir, folder_path) + artist_sanitized = _sanitize_filename(artist_name) + album_sanitized = _sanitize_filename(album_name) + return os.path.join(transfer_dir, artist_sanitized, f"{artist_sanitized} - {album_sanitized}") + + template_context = { + 'artist': 'placeholder', + 'albumartist': 'placeholder', + 'album': 'placeholder', + 'title': 'placeholder', + 'playlist_name': playlist_name, + 'track_number': 1, + 'disc_number': 1, + 'year': '', + 'quality': '' + } + folder_path, _ = get_file_path_from_template(template_context, 'playlist_path') + if folder_path: + return os.path.join(transfer_dir, folder_path) + playlist_sanitized = _sanitize_filename(playlist_name) + return os.path.join(transfer_dir, playlist_sanitized) + +# --- Restored Web UI Helper State --- +session_completed_downloads = 0 +session_stats_lock = threading.Lock() + +batch_locks = {} +_processed_download_ids = set() +_orphaned_download_keys = set() + +_mb_release_cache = {} +_mb_release_cache_lock = threading.Lock() +_mb_release_detail_cache = {} +_mb_release_detail_cache_lock = threading.Lock() + +_enrichment_activity_log = {} +_idle_since = {} +_IDLE_GRACE_SECONDS = 5 + +_status_cache = { + 'spotify': {'connected': False, 'response_time': 0, 'source': 'itunes'}, + 'media_server': {'connected': False, 'response_time': 0, 'type': None}, + 'soulseek': {'connected': False, 'response_time': 0}, +} +_status_cache_timestamps = { + 'spotify': 0, + 'media_server': 0, + 'soulseek': 0, +} +STATUS_CACHE_TTL = 120 + +dev_mode_enabled = False +_hydrabase_ws = None +_hydrabase_lock = threading.Lock() + + +def get_cached_beatport_data(section_type, data_key, genre_slug=None): + current_time = time.time() + with beatport_data_cache['cache_lock']: + try: + if section_type == 'homepage': + cache_entry = beatport_data_cache['homepage'].get(data_key) + elif section_type == 'genre' and genre_slug: + cache_entry = beatport_data_cache['genre'].get(genre_slug, {}).get(data_key) + else: + return None + + if not cache_entry: + return None + + age = current_time - cache_entry['timestamp'] + if age < cache_entry['ttl'] and cache_entry['data'] is not None: + return cache_entry['data'] + return None + except Exception as e: + print(f"Cache lookup error for {section_type}/{data_key}: {e}") + return None + + +def set_cached_beatport_data(section_type, data_key, data, genre_slug=None): + current_time = time.time() + with beatport_data_cache['cache_lock']: + try: + if section_type == 'homepage': + if data_key in beatport_data_cache['homepage']: + beatport_data_cache['homepage'][data_key]['data'] = data + beatport_data_cache['homepage'][data_key]['timestamp'] = current_time + elif section_type == 'genre' and genre_slug: + if genre_slug not in beatport_data_cache['genre']: + beatport_data_cache['genre'][genre_slug] = {} + if data_key not in beatport_data_cache['genre'][genre_slug]: + beatport_data_cache['genre'][genre_slug][data_key] = { + 'data': None, + 'timestamp': 0, + 'ttl': 600, + } + beatport_data_cache['genre'][genre_slug][data_key]['data'] = data + beatport_data_cache['genre'][genre_slug][data_key]['timestamp'] = current_time + except Exception as e: + print(f"Cache storage error for {section_type}/{data_key}: {e}") + + +def add_cache_headers(response, cache_duration=300): + response.headers['Cache-Control'] = f'public, max-age={cache_duration}' + response.headers['Pragma'] = 'cache' + return response + + +def _get_max_concurrent(): + return config_manager.get('download_source.max_concurrent', 3) + + +def _get_batch_max_concurrent(is_album=False, source=None): + if is_album and source in ('soulseek', None): + if source == 'soulseek': + return 1 + mode = config_manager.get('download_source.mode', 'soulseek') + if mode == 'soulseek': + return 1 + return _get_max_concurrent() + + +def _update_task_status(task_id, new_status): + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = new_status + download_tasks[task_id]['status_change_time'] = time.time() + + +class WebUIDownloadMonitor: + def __init__(self): + self.monitoring = False + self.monitor_thread = None + self.monitored_batches = set() + self._lock = threading.Lock() + + def start_monitoring(self, batch_id): + with self._lock: + self.monitored_batches.add(batch_id) + if not self.monitoring: + self.monitoring = True + self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True) + self.monitor_thread.start() + + def stop_monitoring(self, batch_id): + with self._lock: + self.monitored_batches.discard(batch_id) + if not self.monitored_batches: + self.monitoring = False + + def shutdown(self): + with self._lock: + self.monitoring = False + self.monitored_batches.clear() + self.monitor_thread = None + + def _monitor_loop(self): + while self.monitoring and self.monitored_batches: + try: + if globals().get('IS_SHUTTING_DOWN', False): + self.monitoring = False + break + self._check_all_downloads() + time.sleep(1) + except Exception as e: + if 'interpreter shutdown' in str(e) or 'cannot schedule new futures' in str(e): + self.monitoring = False + break + print(f"Download monitor error: {e}") + + def _check_all_downloads(self): + current_time = time.time() + live_transfers_lookup = self._get_live_transfers() + exhausted_tasks = [] + completed_tasks = [] + deferred_ops = [] + + with tasks_lock: + for batch_id in list(self.monitored_batches): + if batch_id not in download_batches: + self.monitored_batches.discard(batch_id) + continue + + for task_id in download_batches[batch_id].get('queue', []): + task = download_tasks.get(task_id) + if not task or task['status'] not in ['downloading', 'queued']: + continue + + retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) + if retry_exhausted: + exhausted_tasks.append((batch_id, task_id)) + + task_filename = task.get('filename') or task.get('track_info', {}).get('filename') + task_username = task.get('username') or task.get('track_info', {}).get('username') + if task_filename and task_username: + lookup_key = _make_context_key(task_username, task_filename) + live_info = live_transfers_lookup.get(lookup_key) + if live_info: + state = live_info.get('state', '') + has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) + has_completion = ('Completed' in state or 'Succeeded' in state) + if has_completion and not has_error: + expected_size = live_info.get('size', 0) + transferred = live_info.get('bytesTransferred', 0) + if expected_size > 0 and transferred < expected_size: + if not task.get('_incomplete_warned'): + task['_incomplete_warned'] = True + continue + if has_completion and not has_error and task['status'] == 'downloading': + task.pop('_incomplete_warned', None) + task['status'] = 'post_processing' + task['status_change_time'] = current_time + completed_tasks.append((batch_id, task_id)) + + for op in deferred_ops: + try: + if op[0] == 'cancel_download': + _, download_id, username = op + run_async(soulseek_client.cancel_download(download_id, username, remove=True)) + elif op[0] == 'cleanup_orphan': + _, context_key = op + with matched_context_lock: + matched_downloads_context.pop(context_key, None) + elif op[0] == 'restart_worker': + _, task_id, batch_id = op + missing_download_executor.submit(_download_track_worker, task_id, batch_id) + except Exception as e: + print(f"[Deferred] Error executing deferred operation {op[0]}: {e}") + + for batch_id, task_id in completed_tasks: + try: + missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id) + _on_download_completed(batch_id, task_id, success=True) + except Exception as e: + print(f"[Monitor] Error handling completed task {task_id}: {e}") + + for batch_id, task_id in exhausted_tasks: + try: + _on_download_completed(batch_id, task_id, success=False) + except Exception as e: + print(f"[Monitor] Error handling exhausted task {task_id}: {e}") + + self._validate_worker_counts() + + def _get_live_transfers(self): + try: + if not self.monitoring or not soulseek_client: + return {} + + live_transfers = {} + transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + if transfers_data: + for user_data in transfers_data: + username = user_data.get('username', 'Unknown') + if 'directories' in user_data: + for directory in user_data['directories']: + if 'files' in directory: + for file_info in directory['files']: + key = _make_context_key(username, file_info.get('filename', '')) + live_transfers[key] = file_info + + try: + all_downloads = run_async(soulseek_client.get_all_downloads()) + for download in all_downloads: + if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): + key = _make_context_key(download.username, download.filename) + live_transfers[key] = { + 'id': download.id, + 'filename': download.filename, + 'username': download.username, + 'state': download.state, + 'percentComplete': download.progress, + 'size': download.size, + 'bytesTransferred': download.transferred, + 'averageSpeed': download.speed, + } + except Exception as yt_error: + print(f"Monitor: Could not fetch streaming source downloads: {yt_error}") + + return live_transfers + except Exception as e: + if ('interpreter shutdown' in str(e) or 'cannot schedule new futures' in str(e) or 'Event loop is closed' in str(e)): + self.monitoring = False + return {} + print(f"Monitor: Could not fetch live transfers: {e}") + return {} + + def _should_retry_task(self, task_id, task, live_transfers_lookup, current_time, deferred_ops): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + task_filename = task.get('filename') or ti.get('filename') + task_username = task.get('username') or ti.get('username') + + if not task_filename or not task_username: + return False + + lookup_key = _make_context_key(task_username, task_filename) + live_info = live_transfers_lookup.get(lookup_key) + + if not live_info: + if current_time - task.get('status_change_time', current_time) > 90: + retry_count = task.get('stuck_retry_count', 0) + last_retry = task.get('last_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 30: + task['stuck_retry_count'] = retry_count + 1 + task['last_retry_time'] = current_time + download_id = task.get('download_id') + if task_username and download_id: + deferred_ops.append(('cancel_download', download_id, task_username)) + if task_username and task_filename: + used_sources = task.get('used_sources', set()) + source_key = f"{task_username}_{task_filename}" + used_sources.add(source_key) + task['used_sources'] = used_sources + if task_username and task_filename: + _orphaned_download_keys.add(lookup_key) + deferred_ops.append(('cleanup_orphan', lookup_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count < 3: + return False + else: + track_label = task.get('track_info', {}).get('name', 'Unknown') + tried_sources = task.get('used_sources', set()) + sources_str = f" (tried {len(tried_sources)} source{'s' if len(tried_sources) != 1 else ''})" if tried_sources else '' + task['status'] = 'failed' + task['error_message'] = f'Download disappeared from transfer list 3 times for "{track_label}"{sources_str} — source may be unavailable' + return bool(task.get('batch_id')) + return False + + state_str = live_info.get('state', '') + progress = live_info.get('percentComplete', 0) + + if any(token in state_str for token in ('Errored', 'Failed', 'Rejected', 'TimedOut')): + retry_count = task.get('error_retry_count', 0) + last_retry = task.get('last_error_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 5: + task['error_retry_count'] = retry_count + 1 + task['last_error_retry_time'] = current_time + _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or _ti.get('username') + filename = task.get('filename') or _ti.get('filename') + download_id = task.get('download_id') + if username and download_id: + deferred_ops.append(('cancel_download', download_id, username)) + if username and filename: + used_sources = task.get('used_sources', set()) + used_sources.add(f"{username}_{filename}") + task['used_sources'] = used_sources + if username and filename: + old_context_key = _make_context_key(username, filename) + _orphaned_download_keys.add(old_context_key) + deferred_ops.append(('cleanup_orphan', old_context_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count < 3: + return False + task['status'] = 'failed' + task['error_message'] = f'Soulseek transfer errored 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' + return bool(task.get('batch_id')) + + elif 'Queued' in state_str or task['status'] == 'queued': + if 'queued_start_time' not in task: + task['queued_start_time'] = current_time + return False + queue_time = current_time - task['queued_start_time'] + timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 + if queue_time > timeout_threshold: + retry_count = task.get('stuck_retry_count', 0) + last_retry = task.get('last_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 30: + task['stuck_retry_count'] = retry_count + 1 + task['last_retry_time'] = current_time + _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or _ti.get('username') + filename = task.get('filename') or _ti.get('filename') + download_id = task.get('download_id') + if username and download_id: + deferred_ops.append(('cancel_download', download_id, username)) + if username and filename: + used_sources = task.get('used_sources', set()) + used_sources.add(f"{username}_{filename}") + task['used_sources'] = used_sources + if username and filename: + old_context_key = _make_context_key(username, filename) + _orphaned_download_keys.add(old_context_key) + deferred_ops.append(('cleanup_orphan', old_context_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count < 3: + return False + task['status'] = 'failed' + task['error_message'] = f'Download stayed queued too long 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + return bool(task.get('batch_id')) + + elif 'InProgress' in state_str and progress < 1: + if 'downloading_start_time' not in task: + task['downloading_start_time'] = current_time + return False + download_time = current_time - task['downloading_start_time'] + timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 + if download_time > timeout_threshold: + retry_count = task.get('stuck_retry_count', 0) + last_retry = task.get('last_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 30: + task['stuck_retry_count'] = retry_count + 1 + task['last_retry_time'] = current_time + _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or _ti.get('username') + filename = task.get('filename') or _ti.get('filename') + download_id = task.get('download_id') + if username and download_id: + deferred_ops.append(('cancel_download', download_id, username)) + if username and filename: + used_sources = task.get('used_sources', set()) + used_sources.add(f"{username}_{filename}") + task['used_sources'] = used_sources + if username and filename: + old_context_key = _make_context_key(username, filename) + _orphaned_download_keys.add(old_context_key) + deferred_ops.append(('cleanup_orphan', old_context_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count < 3: + return False + task['status'] = 'failed' + task['error_message'] = f'Download stuck at 0% three times for "{task.get("track_info", {}).get("name", "Unknown")}"' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + return bool(task.get('batch_id')) + else: + bytes_transferred = live_info.get('bytesTransferred', 0) + if progress >= 1 or bytes_transferred > 0: + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task.pop('stuck_retry_count', None) + else: + if 'downloading_start_time' not in task: + task['downloading_start_time'] = current_time + download_time = current_time - task['downloading_start_time'] + timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 + if download_time > timeout_threshold: + retry_count = task.get('stuck_retry_count', 0) + last_retry = task.get('last_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 30: + task['stuck_retry_count'] = retry_count + 1 + task['last_retry_time'] = current_time + _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or _ti.get('username') + filename = task.get('filename') or _ti.get('filename') + download_id = task.get('download_id') + if username and download_id: + deferred_ops.append(('cancel_download', download_id, username)) + if username and filename: + used_sources = task.get('used_sources', set()) + used_sources.add(f"{username}_{filename}") + task['used_sources'] = used_sources + if username and filename: + old_context_key = _make_context_key(username, filename) + _orphaned_download_keys.add(old_context_key) + deferred_ops.append(('cleanup_orphan', old_context_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count >= 3: + task['status'] = 'failed' + task['error_message'] = f'Download stuck in "{state_str}" state 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + return bool(task.get('batch_id')) + + return False + + def _validate_worker_counts(self): + try: + batches_needing_workers = [] + with tasks_lock: + for batch_id in list(self.monitored_batches): + if batch_id not in download_batches: + continue + batch = download_batches[batch_id] + reported_active = batch['active_count'] + max_concurrent = batch['max_concurrent'] + queue = batch.get('queue', []) + queue_index = batch.get('queue_index', 0) + actually_active = 0 + orphaned_tasks = [] + completed_task_ids = batch.get('_completed_task_ids', set()) + for task_id in queue: + if task_id in download_tasks: + task_status = download_tasks[task_id]['status'] + if task_status in ['searching', 'downloading', 'queued', 'post_processing']: + if task_id not in completed_task_ids: + actually_active += 1 + elif task_status in ['failed', 'completed', 'cancelled', 'not_found'] and task_id in queue[queue_index:]: + orphaned_tasks.append(task_id) + if reported_active != actually_active or orphaned_tasks: + if reported_active != actually_active: + batch['active_count'] = actually_active + if actually_active < max_concurrent and queue_index < len(queue): + batches_needing_workers.append(batch_id) + for batch_id in batches_needing_workers: + try: + _start_next_batch_of_downloads(batch_id) + except Exception as e: + print(f"[Worker Validation] Error starting workers for {batch_id}: {e}") + except Exception as validation_error: + print(f"Error in worker count validation: {validation_error}") + + +download_monitor = WebUIDownloadMonitor() + + +def _is_explicit_blocked(track_data): + if config_manager.get('content_filter.allow_explicit', True): + return False + if track_data.get('explicit', False): + return True + sp_data = track_data.get('spotify_data', {}) + if isinstance(sp_data, str): + try: + sp_data = json.loads(sp_data) + except Exception: + sp_data = {} + return sp_data.get('explicit', False) + + +def fix_artist_image_url(thumb_url): + if not thumb_url: + return None + + try: + needs_fixing = ( + thumb_url.startswith('http://localhost:') or + thumb_url.startswith('https://localhost:') or + thumb_url.startswith('/library/') or + thumb_url.startswith('/Items/') or + thumb_url.startswith('/api/') or + thumb_url.startswith('/rest/') + ) + + if needs_fixing: + active_server = config_manager.get_active_media_server() + if active_server == 'plex': + plex_config = config_manager.get_plex_config() + plex_base_url = plex_config.get('base_url', '') + plex_token = plex_config.get('token', '') + if plex_base_url and plex_token: + if thumb_url.startswith('/library/'): + path = thumb_url + else: + from urllib.parse import urlparse + path = urlparse(thumb_url).path + return f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}" + + elif active_server == 'jellyfin': + jellyfin_config = config_manager.get_jellyfin_config() + jellyfin_base_url = jellyfin_config.get('base_url', '') + jellyfin_token = jellyfin_config.get('api_key', '') + if jellyfin_base_url: + if thumb_url.startswith('/Items/') or thumb_url.startswith('/api/'): + path = thumb_url + else: + from urllib.parse import urlparse + path = urlparse(thumb_url).path + if jellyfin_token: + separator = '&' if '?' in path else '?' + return f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}" + return f"{jellyfin_base_url.rstrip('/')}{path}" + + elif active_server == 'navidrome': + navidrome_config = config_manager.get_navidrome_config() + navidrome_base_url = navidrome_config.get('base_url', '') + navidrome_username = navidrome_config.get('username', '') + navidrome_password = navidrome_config.get('password', '') + if navidrome_base_url and navidrome_username and navidrome_password: + if thumb_url.startswith('/rest/'): + path = thumb_url + else: + from urllib.parse import urlparse + path = urlparse(thumb_url).path + import hashlib + import secrets + salt = secrets.token_hex(6) + token = hashlib.md5((navidrome_password + salt).encode()).hexdigest() + separator = '&' if '?' in path else '?' + auth_params = f"u={navidrome_username}&t={token}&s={salt}&v=1.16.1&c=SoulSync&f=json" + return f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}" + + return thumb_url + except Exception as e: + print(f"Error fixing image URL '{thumb_url}': {e}") + return thumb_url + + +def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): + import os + import re + from difflib import SequenceMatcher + from unidecode import unidecode + + if '||' in api_filename: + _, title = api_filename.split('||', 1) + api_filename = title + + def normalize_for_finding(text: str) -> str: + if not text: + return "" + text = unidecode(text).lower() + text = re.sub(r'[._/]', ' ', text) + text = re.sub(r'[\[\(].*?[\]\)]', '', text) + text = re.sub(r'[^a-z0-9\s-]', '', text) + return ' '.join(text.split()).strip() + + def _path_matches_api_dirs(file_path): + path_parts = set(p.lower() for p in file_path.replace('\\', '/').split('/')) + return all(d in path_parts for d in api_dir_parts) + + def search_in_directory(search_dir, location_name): + best_fuzzy_path = None + highest_fuzzy_similarity = 0.0 + exact_matches = [] + for root, dirs, files in os.walk(search_dir): + dirs[:] = [d for d in dirs if d != 'ss_quarantine'] + for file in files: + if os.path.basename(file) == target_basename: + file_path = os.path.join(root, file) + if api_dir_parts and _path_matches_api_dirs(file_path): + return file_path, 1.0 + if not api_dir_parts: + return file_path, 1.0 + exact_matches.append(file_path) + continue + + file_stem, file_ext_part = os.path.splitext(file) + stripped_stem = re.sub(r'_\d{10,}$', '', file_stem) + if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename: + file_path = os.path.join(root, file) + if api_dir_parts and _path_matches_api_dirs(file_path): + return file_path, 1.0 + if not api_dir_parts: + return file_path, 1.0 + exact_matches.append(file_path) + continue + + normalized_file = normalize_for_finding(file) + similarity = SequenceMatcher(None, normalized_target, normalized_file).ratio() + if similarity > highest_fuzzy_similarity: + highest_fuzzy_similarity = similarity + best_fuzzy_path = os.path.join(root, file) + + if exact_matches: + if len(exact_matches) == 1: + return exact_matches[0], 1.0 + best = exact_matches[0] + best_score = -1 + for m in exact_matches: + m_parts = set(p.lower() for p in m.replace('\\', '/').split('/')) + score = sum(1 for d in api_dir_parts if d in m_parts) + if score > best_score: + best_score = score + best = m + return best, 1.0 + + return best_fuzzy_path, highest_fuzzy_similarity + + target_basename = extract_filename(api_filename) + normalized_target = normalize_for_finding(target_basename) + api_path_normalized = api_filename.replace('\\', '/') if api_filename else '' + api_dir_parts = [p.lower() for p in api_path_normalized.split('/')[:-1] if p] + + best_downloads_path, downloads_similarity = search_in_directory(download_dir, 'downloads') + if downloads_similarity > 0.85: + return (best_downloads_path, 'downloads') + + transfer_similarity = 0.0 + if transfer_dir and os.path.exists(transfer_dir): + best_transfer_path, transfer_similarity = search_in_directory(transfer_dir, 'transfer') + if transfer_similarity > 0.85: + return (best_transfer_path, 'transfer') + + return (None, None) + + +def _cleanup_empty_directories(download_path, moved_file_path): + try: + current_dir = os.path.dirname(moved_file_path) + while current_dir != download_path and current_dir.startswith(download_path): + is_empty = not any(not f.startswith('.') for f in os.listdir(current_dir)) + if is_empty: + os.rmdir(current_dir) + current_dir = os.path.dirname(current_dir) + else: + break + except Exception as e: + print(f"Warning: An error occurred during directory cleanup: {e}") + + +def _sweep_empty_download_directories(): + try: + download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + if not os.path.isdir(download_path): + return 0 + + removed = 0 + for dirpath, _dirnames, _filenames in os.walk(download_path, topdown=False): + if os.path.normpath(dirpath) == os.path.normpath(download_path): + continue + try: + entries = os.listdir(dirpath) + except OSError: + continue + visible = [e for e in entries if not e.startswith('.')] + if not visible: + try: + for hidden in entries: + try: + os.remove(os.path.join(dirpath, hidden)) + except Exception: + pass + os.rmdir(dirpath) + removed += 1 + except OSError: + pass + + return removed + except Exception as e: + print(f"[Folder Cleanup] Error sweeping empty directories: {e}") + return 0 + + +def _get_audio_quality_string(file_path): + try: + ext = os.path.splitext(file_path)[1].lower() + if ext == '.flac': + audio = FLAC(file_path) + return f"FLAC {audio.info.bits_per_sample}bit" + if ext == '.mp3': + from mutagen.mp3 import MP3, BitrateMode + audio = MP3(file_path) + bitrate_kbps = audio.info.bitrate // 1000 + if audio.info.bitrate_mode == BitrateMode.VBR: + return "MP3-VBR" + return f"MP3-{bitrate_kbps}" + if ext in ('.m4a', '.aac', '.mp4'): + audio = MP4(file_path) + return f"M4A-{audio.info.bitrate // 1000}" + if ext == '.ogg': + audio = OggVorbis(file_path) + return f"OGG-{audio.info.bitrate // 1000}" + if ext == '.opus': + from mutagen.oggopus import OggOpus + audio = OggOpus(file_path) + return f"OPUS-{audio.info.bitrate // 1000}" + return '' + except Exception as e: + logger.debug(f"Could not determine audio quality for {file_path}: {e}") + return '' + + +def parse_youtube_playlist(url): + try: + import re + + def clean_youtube_artist(artist_string): + if not artist_string: + return artist_string + original_artist = artist_string + artist_string = artist_string.replace('"', '').replace("'", '') + artist_string = re.sub(r'\s*\([^)]*\)', '', artist_string).strip() + artist_string = re.sub(r'\s*\[[^\]]*\]', '', artist_string).strip() + for suffix in [ + r'\s*-\s*Topic\s*$', + r'\s*VEVO\s*$', + r'\s*Music\s*$', + r'\s*Official\s*$', + r'\s*Records\s*$', + r'\s*Entertainment\s*$', + r'\s*TV\s*$', + r'\s*Channel\s*$', + ]: + artist_string = re.sub(suffix, '', artist_string, flags=re.IGNORECASE).strip() + for sep in [',', '&', ' and ', ' x ', ' X ', ' feat.', ' ft.', ' featuring', ' with', ' vs ', ' vs.']: + if sep in artist_string: + artist_string = artist_string.split(sep)[0].strip() + break + artist_string = re.sub(r'\s+', ' ', artist_string).strip() + artist_string = re.sub(r'^\-\s*|\s*\-$', '', artist_string).strip() + artist_string = re.sub(r'^,\s*|\s*,$', '', artist_string).strip() + return artist_string or original_artist + + def clean_youtube_track_title(title, artist_name=None): + if not title: + return title + original_title = title + artist_removed = False + if artist_name and '-' in title: + artist_pattern = r'^' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*[-–—]\s*' + cleaned_title = re.sub(artist_pattern, '', title, flags=re.IGNORECASE).strip() + if cleaned_title != title: + title = cleaned_title + artist_removed = True + else: + artist_end_pattern = r'\s*[-–—]\s*' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*$' + cleaned_title = re.sub(artist_end_pattern, '', title, flags=re.IGNORECASE).strip() + if cleaned_title != title: + title = cleaned_title + artist_removed = True + title = re.sub(r'【[^】]*】', '', title) + title = re.sub(r'\s*\([^)]*\)', '', title) + title = re.sub(r'\s*\(.*$', '', title) + title = re.sub(r'\[[^\]]*\]', '', title) + title = re.sub(r'\{[^}]*\}', '', title) + title = re.sub(r'<[^>]*>', '', title) + if artist_removed: + title = re.sub(r'\s*-\s*.*$', '', title) + title = re.split(r'\s*\|\s*', title)[0].strip() + for pattern in [ + r'\bapple\s+music\b', + r'\bfull\s+video\b', + r'\bmusic\s+video\b', + r'\bofficial\s+video\b', + r'\bofficial\s+music\s+video\b', + r'\bofficial\b', + r'\bcensored\s+version\b', + r'\buncensored\s+version\b', + r'\bexplicit\s+version\b', + r'\blive\s+version\b', + r'\bversion\b', + r'\btopic\b', + r'\baudio\b', + r'\blyrics?\b', + r'\blyric\s+video\b', + r'\bwith\s+lyrics?\b', + r'\bvisuali[sz]er\b', + r'\bmv\b', + r'\bdirectors?\s+cut\b', + r'\bremaster(ed)?\b', + r'\bremix\b', + ]: + title = re.sub(pattern, '', title, flags=re.IGNORECASE) + if artist_name: + collab_pattern = rf'\b{re.escape(artist_name)}\s*[&,]\s*\w+|[\w\s]+[&,]\s*{re.escape(artist_name)}\b' + if not re.search(collab_pattern, title, flags=re.IGNORECASE): + title = re.sub(rf'\b{re.escape(artist_name)}\b', '', title, flags=re.IGNORECASE) + title = re.sub(rf'\b{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE) + title = re.sub(rf'^{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE) + title = re.sub(r'\s+prod\.?\s+\S+', '', title, flags=re.IGNORECASE) + title = re.sub(r'["\']', '', title) + for pattern in [ + r'\s+feat\.?\s+.+$', + r'\s+ft\.?\s+.+$', + r'\s+featuring\s+.+$', + r'\s+with\s+.+$', + ]: + title = re.sub(pattern, '', title, flags=re.IGNORECASE).strip() + title = re.sub(r'\s+', ' ', title).strip() + title = re.sub(r'^[-–—:,.\s]+|[-–—:,.\s]+$', '', title).strip() + return title or original_title + + ydl_opts = { + 'quiet': True, + 'no_warnings': True, + 'extract_flat': 'in_playlist', + 'skip_download': True, + 'lazy_playlist': False, + } + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + playlist_info = ydl.extract_info(url, download=False) + if not playlist_info: + return None + + playlist_name = playlist_info.get('title', 'Unknown Playlist') + playlist_id = playlist_info.get('id', 'unknown_id') + entries = list(playlist_info.get('entries', []) or []) + tracks = [] + + for entry in entries: + if not entry: + continue + raw_title = entry.get('title', 'Unknown Track') + raw_uploader = entry.get('uploader', 'Unknown Artist') + duration = entry.get('duration', 0) + video_id = entry.get('id', '') + cleaned_artist = clean_youtube_artist(raw_uploader) + cleaned_title = clean_youtube_track_title(raw_title, cleaned_artist) + tracks.append({ + 'id': video_id, + 'name': cleaned_title, + 'artists': [cleaned_artist], + 'duration_ms': duration * 1000 if duration else 0, + 'raw_title': raw_title, + 'raw_artist': raw_uploader, + 'url': f"https://www.youtube.com/watch?v={video_id}", + }) + + return { + 'id': playlist_id, + 'name': playlist_name, + 'tracks': tracks, + 'track_count': len(tracks), + 'url': url, + 'source': 'youtube', + } + except Exception as e: + print(f"Error parsing YouTube playlist: {e}") + return None + + +def get_download_status(): + if not soulseek_client: + return jsonify({"transfers": []}) + + try: + global _processed_download_ids + transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + all_transfers = [] + completed_matched_downloads = [] + _files_claimed_this_cycle = set() + + if transfers_data: + for user_data in transfers_data: + username = user_data.get('username', 'Unknown') + if 'directories' in user_data: + for directory in user_data['directories']: + if 'files' in directory: + for file_info in directory['files']: + file_info['username'] = username + all_transfers.append(file_info) + state = file_info.get('state', '').lower() + if ('succeeded' in state or 'completed' in state) and 'errored' not in state and 'rejected' not in state: + _fi_size = file_info.get('size', 0) + _fi_transferred = file_info.get('bytesTransferred', 0) + if _fi_size > 0 and _fi_transferred < _fi_size: + continue + filename_from_api = file_info.get('filename') + if not filename_from_api: + continue + + context_key = _make_context_key(username, filename_from_api) + if context_key in _orphaned_download_keys: + with matched_context_lock: + has_active_context = context_key in matched_downloads_context + if has_active_context: + _orphaned_download_keys.discard(context_key) + else: + download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + found_result = _find_completed_file_robust(download_dir, filename_from_api) + found_path = found_result[0] if found_result and found_result[0] else None + orphan_cleaned = False + if found_path: + try: + os.remove(found_path) + orphan_cleaned = True + except Exception as e: + print(f"Failed to delete orphaned file (will retry next poll): {e}") + else: + orphan_cleaned = True + if orphan_cleaned: + transfer_id = file_info.get('id') + if transfer_id: + try: + run_async(soulseek_client.cancel_download(str(transfer_id), username, remove=True)) + except Exception: + pass + _orphaned_download_keys.discard(context_key) + continue + + if context_key in _processed_download_ids: + continue + + with matched_context_lock: + context = matched_downloads_context.get(context_key) + available_keys = list(matched_downloads_context.keys())[:5] if not context else None + + if context and context_key not in _stale_transfer_keys: + pass + + if context: + download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + found_result = _find_completed_file_robust(download_dir, filename_from_api) + found_path = found_result[0] if found_result and found_result[0] else None + + if found_path: + _norm_path = os.path.normpath(found_path) + if _norm_path not in _files_claimed_this_cycle: + _files_claimed_this_cycle.add(_norm_path) + completed_matched_downloads.append((context_key, context, found_path)) + with _download_retry_lock: + if context_key in _download_retry_attempts: + del _download_retry_attempts[context_key] + else: + with _download_retry_lock: + if context_key not in _download_retry_attempts: + _download_retry_attempts[context_key] = {'count': 1, 'first_attempt': time.time()} + else: + _download_retry_attempts[context_key]['count'] += 1 + retry_count = _download_retry_attempts[context_key]['count'] + if retry_count >= _download_retry_max: + _processed_download_ids.add(context_key) + del _download_retry_attempts[context_key] + if completed_matched_downloads: + def process_completed_downloads(): + for context_key, context, found_path in completed_matched_downloads: + try: + _pp_task_id = context.get('task_id') + _pp_batch_id = context.get('batch_id') + if _pp_task_id and _pp_batch_id: + _pp_target = _post_process_matched_download_with_verification + _pp_args = (context_key, context, found_path, _pp_task_id, _pp_batch_id) + else: + _pp_target = _post_process_matched_download + _pp_args = (context_key, context, found_path) + thread = threading.Thread(target=_pp_target, args=_pp_args) + thread.daemon = True + thread.start() + _processed_download_ids.add(context_key) + except Exception as e: + print(f"Error starting post-processing thread for {context_key}: {e}") + processing_thread = threading.Thread(target=process_completed_downloads) + processing_thread.daemon = True + processing_thread.start() + + return jsonify({"transfers": all_transfers}) + except Exception as e: + logger.error(f"Error building download status: {e}") + return jsonify({"error": str(e)}), 500 + + +def _get_windowed_calls(key, current_total): + now = time.time() + history = _enrichment_activity_log.setdefault(key, collections.deque(maxlen=17300)) + history.append((now, current_total)) + + cutoff_1h = now - 3600 + cutoff_24h = now - 86400 + oldest_1h_total = current_total + oldest_24h_total = current_total + found_24h = False + for ts, total in history: + if not found_24h and ts >= cutoff_24h: + oldest_24h_total = total + found_24h = True + if ts >= cutoff_1h: + oldest_1h_total = total + break + + return max(0, current_total - oldest_1h_total), max(0, current_total - oldest_24h_total) + + +def _get_enrichment_status(): + services = {} + workers_info = [ + ('musicbrainz', 'MusicBrainz', lambda: mb_worker), + ('spotify_enrichment', 'Spotify', lambda: spotify_enrichment_worker), + ('itunes_enrichment', 'iTunes', lambda: itunes_enrichment_worker), + ('deezer_enrichment', 'Deezer', lambda: deezer_worker), + ('tidal_enrichment', 'Tidal', lambda: tidal_enrichment_worker), + ('qobuz_enrichment', 'Qobuz', lambda: qobuz_enrichment_worker), + ('lastfm', 'Last.fm', lambda: lastfm_worker), + ('genius', 'Genius', lambda: genius_worker), + ('audiodb', 'AudioDB', lambda: audiodb_worker), + ('discogs', 'Discogs', lambda: discogs_worker), + ] + configured_checks = { + 'spotify_enrichment': lambda: bool(config_manager.get('spotify.client_id') and config_manager.get('spotify.client_secret')), + 'tidal_enrichment': lambda: bool(tidal_client and getattr(tidal_client, 'access_token', None)), + 'qobuz_enrichment': lambda: bool(qobuz_enrichment_worker and qobuz_enrichment_worker.client and qobuz_enrichment_worker.client.user_auth_token), + 'lastfm': lambda: bool(config_manager.get('lastfm.api_key', '')), + 'genius': lambda: bool(config_manager.get('genius.access_token', '')), + } + + for key, name, get_worker in workers_info: + worker = get_worker() + if worker is not None: + is_alive = worker.thread is not None and worker.thread.is_alive() + try: + configured = configured_checks.get(key, lambda: True)() + except Exception: + configured = False + + stats = worker.stats + total_processed = stats.get('matched', 0) + stats.get('not_found', 0) + stats.get('errors', 0) + calls_1h, calls_24h = _get_windowed_calls(key, total_processed) + + has_item = getattr(worker, 'current_item', None) is not None + if has_item: + _idle_since.pop(key, None) + is_idle = False + else: + if key not in _idle_since: + _idle_since[key] = time.time() + is_idle = (time.time() - _idle_since[key]) >= _IDLE_GRACE_SECONDS + + svc_data = { + 'name': name, + 'configured': configured, + 'running': worker.running and is_alive and not worker.paused, + 'paused': worker.paused, + 'idle': is_alive and not worker.paused and is_idle, + 'calls_1h': calls_1h, + 'calls_24h': calls_24h, + } + if key == 'spotify_enrichment': + try: + svc_data['daily_budget'] = worker._get_daily_budget_info() + except Exception: + pass + services[key] = svc_data + else: + services[key] = { + 'name': name, + 'configured': False, + 'running': False, + 'paused': False, + 'idle': False, + 'calls_1h': 0, + 'calls_24h': 0, + } + + services['acoustid'] = { + 'name': 'AcoustID', + 'configured': bool(config_manager.get('acoustid.api_key', '')), + } + services['listenbrainz'] = { + 'name': 'ListenBrainz', + 'configured': bool(config_manager.get('listenbrainz.token', '')), + } + return services + + +def _build_system_stats(): + try: + import psutil + except Exception: + psutil = None + + from datetime import timedelta + + start_time = getattr(app, 'start_time', time.time()) + uptime = str(timedelta(seconds=int(time.time() - start_time))) + memory_usage = '0%' + if psutil is not None: + memory_usage = f"{psutil.virtual_memory().percent}%" + + active_downloads = len([batch_id for batch_id, batch_data in download_batches.items() if batch_data.get('phase') == 'downloading']) + with session_stats_lock: + finished_downloads = session_completed_downloads + + total_download_speed = 0.0 + try: + transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + if transfers_data: + for user_data in transfers_data: + if 'directories' in user_data: + for directory in user_data['directories']: + if 'files' in directory: + for file_info in directory['files']: + state = file_info.get('state', '').lower() + if 'inprogress' in state or 'downloading' in state or 'transferring' in state: + speed = file_info.get('averageSpeed', 0) + if isinstance(speed, (int, float)) and speed > 0: + total_download_speed += float(speed) + except Exception: + pass + + return { + 'uptime': uptime, + 'memory_usage': memory_usage, + 'active_downloads': active_downloads, + 'finished_downloads': finished_downloads, + 'download_speed': f"{total_download_speed / (1024 * 1024):.1f} MB/s" if total_download_speed > 1024 * 1024 else f"{total_download_speed / 1024:.1f} KB/s", + } + + +def _is_hydrabase_active(): + try: + if hydrabase_client is None or not hydrabase_client.is_connected(): + return False + return dev_mode_enabled + except (NameError, Exception): + return False + # --- Automation Engine --- try: automation_engine = AutomationEngine(get_database()) @@ -658,6 +2217,48 @@ except Exception as e: logger.error(f"Automation engine failed to initialize: {e}") automation_engine = None +# --- Automation Progress Tracking --- +automation_progress_states = {} # automation_id (int) -> state dict +automation_progress_lock = threading.Lock() +_scan_library_automation_id = None + +def _init_automation_progress(automation_id, automation_name, action_type): + """Initialize progress state when an automation starts running.""" + with automation_progress_lock: + automation_progress_states[automation_id] = { + 'status': 'running', + 'action_type': action_type, + 'progress': 0, 'phase': 'Starting...', 'current_item': '', + 'processed': 0, 'total': 0, + 'log': [{'type': 'info', 'text': f'Starting {automation_name}'}], + 'started_at': datetime.now().isoformat(), + 'finished_at': None, + } + + +def _update_automation_progress(automation_id, **kwargs): + """Update progress state from handler threads. Thread-safe.""" + if automation_id is None: + return + with automation_progress_lock: + state = automation_progress_states.get(automation_id) + if not state: + return + for k, v in kwargs.items(): + if k == 'log_line': + state['log'].append({'type': kwargs.get('log_type', 'info'), 'text': v}) + if len(state['log']) > 50: + state['log'] = state['log'][-50:] + elif k != 'log_type': + state[k] = v + # Immediate emit on finish so frontend gets final state without waiting for loop + if kwargs.get('status') in ('finished', 'error'): + state['finished_at'] = datetime.now().isoformat() + try: + socketio.emit('automation:progress', {str(automation_id): dict(state)}) + except Exception: + pass + def _register_automation_handlers(): """Register real SoulSync action handlers with the automation engine.""" if not automation_engine: @@ -18089,31 +19690,6 @@ def _wipe_source_tags(file_path: str) -> bool: def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: return metadata_enrichment.enhance_file_metadata(file_path, context, artist, album_info) -def _get_image_dimensions(data: bytes): - """Extract width/height from JPEG or PNG image data without PIL.""" - try: - if data[:8] == b'\x89PNG\r\n\x1a\n': - # PNG: width and height at bytes 16-23 - import struct - w, h = struct.unpack('>II', data[16:24]) - return w, h - if data[:2] == b'\xff\xd8': - # JPEG: scan for SOF0/SOF2 marker - import struct - i = 2 - while i < len(data) - 9: - if data[i] != 0xFF: - break - marker = data[i + 1] - if marker in (0xC0, 0xC2): - h, w = struct.unpack('>HH', data[i + 5:i + 9]) - return w, h - length = struct.unpack('>H', data[i + 2:i + 4])[0] - i += 2 + length - except Exception: - pass - return None, None - def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): return metadata_enrichment.download_cover_art( @@ -18323,8 +19899,34 @@ def _post_process_matched_download(context_key, context, file_path): from core.import_pipeline import post_process_matched_download return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime()) -# Keep track of processed downloads to avoid re-processing -_processed_download_ids = set() +def _build_import_pipeline_runtime(): + """Collect the live controller dependencies needed by core.import_pipeline.""" + return types.SimpleNamespace( + automation_engine=automation_engine, + on_download_completed=_on_download_completed, + web_scan_manager=web_scan_manager, + repair_worker=repair_worker, + ) + +def _wipe_source_tags(file_path: str) -> bool: + return metadata_enrichment.wipe_source_tags(file_path) + + +def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: + return metadata_enrichment.enhance_file_metadata( + file_path, + context, + artist, + album_info, + ) + + +def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): + return metadata_enrichment.download_cover_art( + album_info, + target_dir, + context, + ) # Track stale transfer keys (completed in slskd but no context — e.g., from before app restart) # so we only log the warning once per key instead of spamming every poll cycle