diff --git a/api/downloads.py b/api/downloads.py index 448b80cd..fd36e91e 100644 --- a/api/downloads.py +++ b/api/downloads.py @@ -5,6 +5,7 @@ Download management endpoints — list, cancel active downloads. from flask import request, current_app from .auth import require_api_key from .helpers import api_success, api_error +from core.import_runtime_state import download_tasks, tasks_lock def _serialize_download(task_id, task): @@ -53,8 +54,6 @@ def register_routes(bp): descending so newest/in-flight tasks appear first. """ try: - from web_server import download_tasks, tasks_lock - # Parse pagination params try: limit = int(request.args.get("limit", 100)) diff --git a/api/system.py b/api/system.py index 4a0ecb03..b6a72141 100644 --- a/api/system.py +++ b/api/system.py @@ -55,7 +55,7 @@ def register_routes(bp): def system_activity(): """Recent activity feed.""" try: - from web_server import activity_feed + from core.import_runtime_state import activity_feed items = list(activity_feed) if activity_feed else [] return api_success({"activities": items}) except Exception as e: @@ -74,7 +74,7 @@ def register_routes(bp): # Active download count download_count = 0 try: - from web_server import download_tasks, tasks_lock + from core.import_runtime_state import download_tasks, tasks_lock with tasks_lock: download_count = sum( 1 for t in download_tasks.values() diff --git a/core/import_album.py b/core/import_album.py new file mode 100644 index 00000000..953e091e --- /dev/null +++ b/core/import_album.py @@ -0,0 +1,515 @@ +"""Album import helpers for staging matching and post-processing context.""" + +from __future__ import annotations + +import os +from typing import Any, Dict, Iterable, List, Optional, Set + +from core.import_context import normalize_import_context +from core.import_file_ops import read_staging_file_metadata +from core.import_staging import AUDIO_EXTENSIONS, get_staging_path +from utils.logging_config import get_logger + + +logger = get_logger("import_album") + +def get_client_for_source(source: str): + from core.metadata_service import get_client_for_source as _get_client_for_source + + return _get_client_for_source(source) + + +def get_artist_album_tracks( + album_id: str, + artist_name: str = "", + album_name: str = "", + source: Optional[str] = None, +): + from core.metadata_service import get_artist_album_tracks as _get_artist_album_tracks + + return _get_artist_album_tracks( + album_id, + artist_name=artist_name, + album_name=album_name, + source_override=source, + ) + + +try: + from core.matching_engine import MusicMatchingEngine + _MATCHING_ENGINE_IMPORT_ERROR = None +except Exception as exc: # pragma: no cover - only hits in stripped-down environments + MusicMatchingEngine = None # type: ignore[assignment] + _MATCHING_ENGINE_IMPORT_ERROR = exc + + +_MATCHING_ENGINE = None + + +def _get_matching_engine() -> Any: + global _MATCHING_ENGINE + if _MATCHING_ENGINE is None: + if MusicMatchingEngine is None: + raise RuntimeError("Music matching engine is unavailable") from _MATCHING_ENGINE_IMPORT_ERROR + _MATCHING_ENGINE = MusicMatchingEngine() + return _MATCHING_ENGINE + + +def _normalize_artist_entries(artists: Any) -> List[Dict[str, Any]]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + normalized: List[Dict[str, Any]] = [] + for artist in artists: + if isinstance(artist, dict): + entry: Dict[str, Any] = {} + name = artist.get("name") or artist.get("artist_name") or artist.get("title") or "" + artist_id = artist.get("id") or artist.get("artist_id") or "" + if name: + entry["name"] = str(name) + if artist_id: + entry["id"] = str(artist_id) + genres = artist.get("genres") + if genres is not None: + entry["genres"] = genres + if entry: + normalized.append(entry) + continue + + name = str(artist).strip() + if name: + normalized.append({"name": name}) + + return normalized + + +def _normalize_album_source(album: Dict[str, Any], source: str = "") -> str: + album_source = source or album.get("source") or "" + return str(album_source).strip().lower() + + +def _strip_legacy_source_fields(payload: Any) -> Any: + if not isinstance(payload, dict): + return payload + + cleaned = dict(payload) + cleaned.pop("_source", None) + cleaned.pop("provider", None) + return cleaned + + +def _extract_track_artist_name(track: Dict[str, Any]) -> str: + artists = track.get("artists") or [] + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + if not artists: + return "" + + first = artists[0] + if isinstance(first, dict): + return str(first.get("name") or first.get("artist_name") or first.get("title") or "").strip() + return str(first or "").strip() + + +def _coerce_track_int(value: Any, default: int = 1) -> int: + if value in (None, ""): + return default + try: + return int(str(value).split("/")[0].strip() or default) + except (TypeError, ValueError): + return default + + +def _collect_staging_files(file_paths: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]: + staging_path = get_staging_path() + file_filter: Optional[Set[str]] = set(file_paths) if file_paths else None + staging_files: List[Dict[str, Any]] = [] + + if not os.path.isdir(staging_path): + return staging_files + + for root, _dirs, filenames in os.walk(staging_path): + for filename in filenames: + ext = os.path.splitext(filename)[1].lower() + if ext not in AUDIO_EXTENSIONS: + continue + + full_path = os.path.join(root, filename) + if file_filter is not None and full_path not in file_filter: + continue + + meta = read_staging_file_metadata(full_path, filename) + staging_files.append( + { + "filename": filename, + "full_path": full_path, + "title": meta.get("title", ""), + "artist": meta.get("albumartist") or meta.get("artist") or "", + "album": meta.get("album", ""), + "albumartist": meta.get("albumartist") or meta.get("artist") or "", + "track_number": meta.get("track_number", 1), + "disc_number": meta.get("disc_number", 1), + } + ) + + return staging_files + + +def _normalize_match_track(track: Dict[str, Any], source: str, album: Dict[str, Any]) -> Dict[str, Any]: + track_album = track.get("album") if isinstance(track.get("album"), dict) else album + if isinstance(track_album, dict): + track_album = _strip_legacy_source_fields(track_album) + track_source = _normalize_album_source(track, source) + track_artists = _normalize_artist_entries(track.get("artists") or []) + + if not track_artists and album.get("artists"): + track_artists = _normalize_artist_entries(album.get("artists")) + + return { + "id": track.get("id", ""), + "name": track.get("name", "Unknown Track"), + "track_number": _coerce_track_int(track.get("track_number", 1), default=1), + "disc_number": _coerce_track_int(track.get("disc_number", 1), default=1), + "duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0), + "artists": track_artists, + "uri": track.get("uri", ""), + "album": track_album, + "source": track_source, + } + + +def _score_album_track_match(track: Dict[str, Any], staging_file: Dict[str, Any], album_name: str) -> float: + engine = _get_matching_engine() + + track_name = track.get("name", "") + staging_title = staging_file.get("title", "") + score = 0.0 + + title_sim = engine.similarity_score( + engine.normalize_string(track_name), + engine.normalize_string(staging_title or ""), + ) + score += title_sim * 0.45 + + track_artist_name = _extract_track_artist_name(track) + staging_artist = staging_file.get("artist") or "" + if track_artist_name and staging_artist: + artist_sim = engine.similarity_score( + engine.normalize_string(track_artist_name), + engine.normalize_string(staging_artist), + ) + score += artist_sim * 0.15 + else: + score += 0.075 + + track_number = _coerce_track_int(track.get("track_number", 1), default=1) + staging_track_number = _coerce_track_int(staging_file.get("track_number", 1), default=1) + if staging_track_number and track_number: + if staging_track_number == track_number: + score += 0.30 + elif abs(staging_track_number - track_number) <= 1: + score += 0.12 + + staging_album = staging_file.get("album") or "" + if staging_album and album_name: + album_sim = engine.similarity_score( + engine.normalize_string(staging_album), + engine.normalize_string(album_name), + ) + score += album_sim * 0.10 + + return score + + +def _fetch_artist_data_for_source(client: Any, artist_id: str, source: str) -> Any: + if source == "spotify": + try: + return client.get_artist(artist_id, allow_fallback=False) + except TypeError: + return client.get_artist(artist_id) + return client.get_artist(artist_id) + + +def resolve_album_artist_context(album: Dict[str, Any], source: str = "") -> Dict[str, Any]: + """Build a neutral artist context for album import processing.""" + album = dict(album or {}) + source = _normalize_album_source(album, source) + + artists = _normalize_artist_entries(album.get("artists") or []) + if not artists: + artist_name = album.get("artist") or album.get("artist_name") or "" + artist_id = album.get("artist_id") or "" + if artist_name or artist_id: + artist_entry: Dict[str, Any] = {} + if artist_name: + artist_entry["name"] = str(artist_name) + if artist_id: + artist_entry["id"] = str(artist_id) + artists = [artist_entry] + + primary_artist = artists[0] if artists else {} + artist_name = str( + primary_artist.get("name") + or album.get("artist") + or album.get("artist_name") + or "Unknown Artist" + ).strip() + artist_id = str(primary_artist.get("id") or album.get("artist_id") or "").strip() + + genres: List[Any] = [] + if artist_id and source: + client = get_client_for_source(source) + if client and hasattr(client, "get_artist"): + try: + artist_data = _fetch_artist_data_for_source(client, artist_id, source) + raw_genres = artist_data.get("genres") if isinstance(artist_data, dict) else getattr(artist_data, "genres", []) + if isinstance(raw_genres, str): + genres = [raw_genres] + elif raw_genres: + try: + genres = list(raw_genres) + except TypeError: + genres = [raw_genres] + except Exception as exc: + logger.debug("Could not resolve artist genres for %s on %s: %s", artist_id, source, exc) + + return { + "id": artist_id, + "name": artist_name, + "genres": genres, + "source": source, + } + + +def build_album_import_context( + album: Dict[str, Any], + track: Dict[str, Any], + *, + artist_context: Optional[Dict[str, Any]] = None, + total_discs: int = 1, + source: str = "", +) -> Dict[str, Any]: + """Build a neutral post-processing context for one album track.""" + album = dict(album or {}) + track = dict(track or {}) + source = _normalize_album_source(album, source) + + album_artists = _normalize_artist_entries(album.get("artists") or []) + if not album_artists and artist_context: + album_artists = _normalize_artist_entries([artist_context]) + + if artist_context: + artist_ctx = dict(artist_context) + else: + artist_ctx = resolve_album_artist_context(album, source) + + artist_ctx = _strip_legacy_source_fields(artist_ctx) + artist_ctx.setdefault("genres", []) + artist_ctx.setdefault("source", source) + artist_ctx["genres"] = artist_ctx.get("genres") or [] + + track_artists = _normalize_artist_entries(track.get("artists") or []) + if not track_artists: + track_artists = album_artists or [artist_ctx] + + track_album_value = track.get("album") + if isinstance(track_album_value, dict): + track_album_name = ( + track_album_value.get("name") + or track_album_value.get("title") + or album.get("name") + or album.get("album_name") + or "" + ) + track_album_id = str(track_album_value.get("id") or track_album_value.get("album_id") or "").strip() + track_album_type = track_album_value.get("album_type") or album.get("album_type") or "album" + track_album_release = track_album_value.get("release_date") or album.get("release_date") or "" + track_album_image = track_album_value.get("image_url") or album.get("image_url") or "" + else: + track_album_name = str(track_album_value or album.get("name") or album.get("album_name") or "").strip() + track_album_id = str(album.get("id") or album.get("album_id") or "").strip() + track_album_type = album.get("album_type") or "album" + track_album_release = album.get("release_date") or "" + track_album_image = album.get("image_url") or "" + + album_name = str(album.get("name") or album.get("album_name") or track_album_name or "Unknown Album").strip() + artist_name = str( + artist_ctx.get("name") + or album.get("artist") + or album.get("artist_name") + or "Unknown Artist" + ).strip() + + track_number = _coerce_track_int(track.get("track_number", 1), default=1) + disc_number = _coerce_track_int(track.get("disc_number", 1), default=1) + + normalized_track = { + "id": str(track.get("id") or track.get("track_id") or "").strip(), + "name": str(track.get("name") or "Unknown Track").strip(), + "track_number": track_number, + "disc_number": disc_number, + "duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0), + "artists": track_artists, + "uri": str(track.get("uri") or "").strip(), + "album": track_album_name, + "album_id": track_album_id, + "album_type": track_album_type, + "release_date": track_album_release, + "source": source, + } + + normalized_album = { + "id": str(album.get("id") or album.get("album_id") or track_album_id or "").strip(), + "name": album_name, + "artist": artist_name, + "artist_name": artist_name, + "artist_id": str(artist_ctx.get("id") or album.get("artist_id") or "").strip(), + "artists": album_artists, + "release_date": str(album.get("release_date") or track_album_release or "").strip(), + "total_tracks": int(album.get("total_tracks") or track.get("total_tracks") or 0) or 1, + "total_discs": int(total_discs or 1) if str(total_discs or 1).isdigit() else total_discs or 1, + "album_type": str(album.get("album_type") or track_album_type or "album").strip() or "album", + "image_url": str(album.get("image_url") or track_album_image or "").strip(), + "images": album.get("images") or ([] if not track_album_image else [{"url": track_album_image}]), + "source": source, + } + + original_search = { + "title": normalized_track["name"], + "artist": artist_name, + "album": album_name, + "track_number": track_number, + "disc_number": disc_number, + "clean_title": normalized_track["name"], + "clean_album": album_name, + "clean_artist": artist_name, + "artists": track_artists, + "duration_ms": normalized_track["duration_ms"], + "id": normalized_track["id"], + "source": source, + } + + context = { + "artist": artist_ctx, + "album": normalized_album, + "track_info": normalized_track, + "original_search_result": original_search, + "is_album_download": True, + "has_clean_metadata": bool(normalized_track["id"]), + "has_full_metadata": bool(normalized_track["id"]), + "source": source, + } + + normalized_context = normalize_import_context(context) + normalized_context["artist"] = _strip_legacy_source_fields(normalized_context.get("artist")) + normalized_context["album"] = _strip_legacy_source_fields(normalized_context.get("album")) + normalized_context["track_info"] = _strip_legacy_source_fields(normalized_context.get("track_info")) + normalized_context["original_search_result"] = _strip_legacy_source_fields(normalized_context.get("original_search_result")) + return normalized_context + + +def build_album_import_match_payload( + album_id: str, + *, + album_name: str = "", + album_artist: str = "", + file_paths: Optional[Iterable[str]] = None, + source: Optional[str] = None, +) -> Dict[str, Any]: + """Build the album import match payload using provider-priority metadata lookup.""" + album_response = get_artist_album_tracks( + album_id, + artist_name=album_artist, + album_name=album_name, + source=source, + ) + + album = _strip_legacy_source_fields(dict(album_response.get("album") or {})) + source = _normalize_album_source(album, album_response.get("source") or source or "") + tracks = list(album_response.get("tracks") or []) + if not album_response.get("success") or not tracks: + return { + "success": False, + "error": album_response.get("error", "Album not found"), + "status_code": album_response.get("status_code", 404), + "album": { + "id": album_id, + "name": album_name or album_id, + "artist": album_artist or "Unknown Artist", + "artist_name": album_artist or "Unknown Artist", + "artist_id": "", + "artists": [], + "release_date": "", + "total_tracks": 0, + "total_discs": 1, + "album_type": "album", + "image_url": "", + "images": [], + "source": source, + }, + "matches": [], + "unmatched_files": [], + "source": source, + "source_priority": album_response.get("source_priority", []), + "resolved_album_id": album_response.get("resolved_album_id") or album_id, + } + + staging_files = _collect_staging_files(file_paths) + album_name_for_match = album.get("name") or album_name or "" + matches: List[Dict[str, Any]] = [] + used_files: Set[int] = set() + + for track in tracks: + normalized_track = _normalize_match_track(track, source, album) + best_match = None + best_score = 0.0 + + for index, staging_file in enumerate(staging_files): + if index in used_files: + continue + + score = _score_album_track_match(normalized_track, staging_file, album_name_for_match) + if score > best_score and score >= 0.4: + best_score = score + best_match = index + + matches.append( + { + "track": normalized_track, + "staging_file": staging_files[best_match] if best_match is not None else None, + "confidence": round(best_score, 2) if best_match is not None else 0, + } + ) + + if best_match is not None: + used_files.add(best_match) + + unmatched_files = [sf for index, sf in enumerate(staging_files) if index not in used_files] + + return { + "success": True, + "album": album, + "matches": matches, + "unmatched_files": unmatched_files, + "source": source, + "source_priority": album_response.get("source_priority", []), + "resolved_album_id": album_response.get("resolved_album_id") or album_id, + } diff --git a/core/import_context.py b/core/import_context.py new file mode 100644 index 00000000..cf9504ff --- /dev/null +++ b/core/import_context.py @@ -0,0 +1,326 @@ +"""Helpers for normalizing and reading import contexts. + +These functions keep the single-import pipeline source-agnostic while still +accepting legacy `spotify_*` payloads from older callers. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + + +def _as_dict(value: Any) -> Dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _first_value(mapping: Dict[str, Any], *keys: str, default: Any = "") -> Any: + for key in keys: + if key in mapping: + value = mapping.get(key) + if value not in (None, ""): + return value + return default + + +def _first_id_value(*values: Any) -> str: + for value in values: + if value in (None, ""): + continue + text = str(value).strip() + if text: + return text + return "" + + +def extract_artist_name(artist: Any) -> str: + if isinstance(artist, dict): + return str(artist.get("name", "") or "") + if hasattr(artist, "name"): + return str(getattr(artist, "name") or "") + return str(artist) if artist else "" + + +def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """Normalize an import context to neutral fields in place and drop legacy aliases.""" + if not isinstance(context, dict): + return {} + + artist = _as_dict(context.get("artist") or context.get("spotify_artist")) + album = _as_dict(context.get("album") or context.get("spotify_album")) + track_info = _as_dict(context.get("track_info")) + original_search = _as_dict(context.get("original_search_result")) + + context["artist"] = artist + context["album"] = album + context["track_info"] = track_info + context["original_search_result"] = original_search + context.pop("spotify_artist", None) + context.pop("spotify_album", None) + + for clean_key, legacy_key in ( + ("clean_title", "spotify_clean_title"), + ("clean_album", "spotify_clean_album"), + ("clean_artist", "spotify_clean_artist"), + ): + if clean_key not in original_search or original_search.get(clean_key) in (None, ""): + legacy_value = original_search.get(legacy_key) + if legacy_value not in (None, ""): + original_search[clean_key] = legacy_value + original_search.pop(legacy_key, None) + + has_clean = bool(context.get("has_clean_metadata", context.get("has_clean_spotify_data", False))) + has_full = bool(context.get("has_full_metadata", context.get("has_full_spotify_metadata", False))) + context["has_clean_metadata"] = has_clean + context["has_full_metadata"] = has_full + context.pop("has_clean_spotify_data", None) + context.pop("has_full_spotify_metadata", None) + + return context + + +def get_import_context_artist(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("artist") or context.get("spotify_artist")) + + +def get_import_context_album(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("album") or context.get("spotify_album")) + + +def get_import_track_info(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("track_info")) + + +def get_import_original_search(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("original_search_result")) + + +def get_import_source(context: Optional[Dict[str, Any]]) -> str: + if not isinstance(context, dict): + return "" + + source = context.get("source") + if source: + return str(source) + + track_info = get_import_track_info(context) + source = _first_value(track_info, "source", default="") + if source: + return str(source) + + original_search = get_import_original_search(context) + source = _first_value(original_search, "source", default="") + if source: + return str(source) + + album = get_import_context_album(context) + source = _first_value(album, "source", default="") + if source: + return str(source) + + artist = get_import_context_artist(context) + source = _first_value(artist, "source", default="") + return str(source) if source else "" + + +def get_import_clean_title( + context: Optional[Dict[str, Any]], + album_info: Optional[Dict[str, Any]] = None, + default: str = "Unknown Track", +) -> str: + original_search = get_import_original_search(context) + title = _first_value( + original_search, + "clean_title", + "title", + default="", + ) + if not title and album_info: + title = _first_value(album_info, "clean_track_name", "track_name", default="") + if not title: + track_info = get_import_track_info(context) + title = _first_value(track_info, "name", "title", default="") + return str(title or default) + + +def get_import_clean_album( + context: Optional[Dict[str, Any]], + album_info: Optional[Dict[str, Any]] = None, + default: str = "Unknown Album", +) -> str: + original_search = get_import_original_search(context) + album = _first_value( + original_search, + "clean_album", + "album", + default="", + ) + if not album and album_info: + album = _first_value(album_info, "album_name", "clean_album_name", default="") + if not album: + album_ctx = get_import_context_album(context) + album = _first_value(album_ctx, "name", default="") + return str(album or default) + + +def get_import_clean_artist(context: Optional[Dict[str, Any]], default: str = "Unknown Artist") -> str: + original_search = get_import_original_search(context) + artist = _first_value( + original_search, + "clean_artist", + "artist", + default="", + ) + if not artist: + artist_ctx = get_import_context_artist(context) + artist = _first_value(artist_ctx, "name", default="") + return str(artist or default) + + +def get_import_has_clean_metadata(context: Optional[Dict[str, Any]]) -> bool: + if not isinstance(context, dict): + return False + return bool(context.get("has_clean_metadata", False)) + + +def get_import_has_full_metadata(context: Optional[Dict[str, Any]]) -> bool: + if not isinstance(context, dict): + return False + return bool(context.get("has_full_metadata", False)) + + +def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist = get_import_context_artist(context) + album = get_import_context_album(context) + + return { + "track_id": _first_id_value( + _first_value(track_info, "id", "track_id", "trackId", "source_track_id", default=""), + _first_value(track_info, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), + _first_value(original_search, "id", "track_id", "source_track_id", default=""), + _first_value(original_search, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), + ), + "artist_id": _first_id_value( + _first_value(artist, "id", "artist_id", "source_artist_id", default=""), + _first_value(artist, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), + _first_value(original_search, "artist_id", "source_artist_id", default=""), + _first_value(original_search, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), + ), + "album_id": _first_id_value( + _first_value(album, "id", "album_id", "collectionId", "source_album_id", default=""), + _first_value(album, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + _first_value(original_search, "album_id", "source_album_id", default=""), + _first_value(original_search, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + _first_value(track_info, "album_id", "source_album_id", default=""), + _first_value(track_info, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + ), + } + + +def get_source_tag_names(source: str) -> Dict[str, Optional[str]]: + source_name = (source or "").strip().lower() + if source_name == "spotify": + return {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"} + if source_name == "itunes": + return {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"} + if source_name == "deezer": + return {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None} + if source_name == "hydrabase": + return {"track": None, "artist": None, "album": None} + if source_name == "discogs": + return {"track": None, "artist": None, "album": None} + return {"track": None, "artist": None, "album": None} + + +def get_library_source_id_columns(source: str) -> Dict[str, Optional[str]]: + source_name = (source or "").strip().lower() + if source_name == "spotify": + return {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"} + if source_name == "itunes": + return {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"} + if source_name == "deezer": + return {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"} + if source_name == "hydrabase": + return {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"} + if source_name == "discogs": + return {"artist": "discogs_id", "album": "discogs_id", "track": None} + return {} + + +def build_import_album_info( + context: Optional[Dict[str, Any]], + *, + album_info: Optional[Dict[str, Any]] = None, + force_album: bool = False, +) -> Dict[str, Any]: + """Build the album-info payload used by post-processing.""" + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist_ctx = get_import_context_artist(context) + + track_number = ( + (album_info or {}).get("track_number") + or track_info.get("track_number") + or original_search.get("track_number") + or 1 + ) + disc_number = ( + (album_info or {}).get("disc_number") + or track_info.get("disc_number") + or original_search.get("disc_number") + or 1 + ) + + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + album_name = get_import_clean_album(context, album_info=album_info, default=original_search.get("album", "Unknown Album")) + album_image_url = ( + (album_info or {}).get("album_image_url") + or album_ctx.get("image_url") + or "" + ) + total_tracks = ( + album_ctx.get("total_tracks") + or track_info.get("total_tracks") + or (album_info or {}).get("total_tracks") + or 0 + ) + album_type = (album_ctx.get("album_type") or track_info.get("album_type") or "album") + source = get_import_source(context) + + artist_name = artist_ctx.get("name") or original_search.get("artist") or get_import_clean_artist(context) + normalized_album = str(album_name or "").strip().lower() + normalized_title = str(clean_track_name or "").strip().lower() + normalized_artist = str(artist_name or "").strip().lower() + is_album = bool( + force_album + or ( + normalized_album + and total_tracks + and int(total_tracks) > 1 + and normalized_album != normalized_title + and normalized_album != normalized_artist + ) + ) + + return { + "is_album": is_album, + "album_name": album_name, + "track_number": int(track_number) if str(track_number).isdigit() else track_number, + "disc_number": int(disc_number) if str(disc_number).isdigit() else disc_number, + "clean_track_name": clean_track_name, + "album_image_url": album_image_url, + "confidence": (album_info or {}).get("confidence", 1.0 if is_album or force_album else 0.0), + "source": source, + "album_type": album_type, + "total_tracks": int(total_tracks) if str(total_tracks).isdigit() else total_tracks, + } diff --git a/core/import_file_ops.py b/core/import_file_ops.py new file mode 100644 index 00000000..261687a8 --- /dev/null +++ b/core/import_file_ops.py @@ -0,0 +1,475 @@ +"""Shared file and path helpers for import processing.""" + +from __future__ import annotations + +import logging +import os +import re +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any, Dict, Optional + +logger = logging.getLogger("import_file_ops") + + +def _get_config_manager(): + from config.settings import config_manager + return config_manager + + +def safe_move_file(src, dst): + """Move a file safely across filesystems.""" + src = Path(src) + dst = Path(dst) + + dst.parent.mkdir(parents=True, exist_ok=True) + + if not src.exists(): + if dst.exists(): + logger.info(f"Source gone but destination exists, file already transferred: {dst.name}") + return + raise FileNotFoundError(f"Source file not found and destination does not exist: {src}") + + if dst.exists(): + for _attempt in range(3): + try: + dst.unlink() + break + except PermissionError: + if _attempt < 2: + time.sleep(1) + else: + logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}") + except Exception: + break + + try: + shutil.move(str(src), str(dst)) + return + except FileNotFoundError: + if dst.exists(): + logger.info(f"Source moved by another thread, destination exists: {dst.name}") + return + raise + except (OSError, PermissionError) as e: + error_msg = str(e).lower() + + if dst.exists() and dst.stat().st_size > 0: + logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}") + try: + src.unlink() + except Exception: + logger.info(f"Could not delete source file (may be owned by another process): {src}") + return + + if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg: + logger.warning(f"Cross-device move detected, using fallback copy method: {e}") + try: + with open(src, "rb") as f_src: + with open(dst, "wb") as f_dst: + shutil.copyfileobj(f_src, f_dst) + f_dst.flush() + os.fsync(f_dst.fileno()) + + try: + src.unlink() + except PermissionError: + logger.info(f"Could not delete source file (may be owned by another process): {src}") + logger.info(f"Successfully moved file using fallback method: {src} -> {dst}") + return + except Exception as fallback_error: + logger.error(f"Fallback copy also failed: {fallback_error}") + raise + raise + + +def extract_track_number_from_filename(filename: str, title: str = None) -> int: + """Extract track number from a filename. Returns 1 if not found.""" + basename = os.path.splitext(os.path.basename(filename))[0].strip() + + match = re.match(r"^\d[\-\.](\d{1,2})\s*[\-\.]\s*", basename) + if match: + num = int(match.group(1)) + if 1 <= num <= 99: + return num + + match = re.match(r"^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*", basename) + if match: + num = int(match.group(1)) + if 1 <= num <= 999: + return num + + return 1 + + +def _coerce_tag_number(value: Any, default: int = 1) -> int: + if value in (None, ""): + return default + + if isinstance(value, (list, tuple)): + value = value[0] if value else None + + if value in (None, ""): + return default + + text = str(value).strip() + if not text: + return default + + match = re.match(r"^(\d+)", text) + if match: + try: + return int(match.group(1)) + except ValueError: + return default + + try: + return int(text) + except (TypeError, ValueError): + return default + + +def read_staging_file_metadata(file_path: str, filename: Optional[str] = None) -> Dict[str, Any]: + """Read common audio tag metadata from a staging file.""" + try: + from mutagen import File as MutagenFile + + tags = MutagenFile(file_path, easy=True) + except Exception: + tags = None + + filename = filename or os.path.basename(file_path) + stem = os.path.splitext(os.path.basename(filename))[0] + + def _first_tag(*keys: str) -> str: + if not tags: + return "" + for key in keys: + try: + value = tags.get(key) # type: ignore[attr-defined] + except Exception: + value = None + if value: + if isinstance(value, (list, tuple)): + value = value[0] if value else "" + text = str(value).strip() + if text: + return text + return "" + + title = _first_tag("title") + artist = _first_tag("artist") + albumartist = _first_tag("albumartist") + album = _first_tag("album") + + if not title: + title = stem + if not albumartist: + albumartist = artist + + track_number = _coerce_tag_number(_first_tag("tracknumber", "track_number"), default=0) + if not track_number: + track_number = extract_track_number_from_filename(filename or file_path) + + disc_number = _coerce_tag_number(_first_tag("discnumber", "disc_number"), default=1) + + return { + "title": title, + "artist": artist, + "albumartist": albumartist, + "album": album, + "track_number": track_number, + "disc_number": disc_number, + } + + +def cleanup_empty_directories(download_path, moved_file_path): + """Remove empty directories after a move, ignoring hidden files.""" + try: + current_dir = os.path.dirname(moved_file_path) + while current_dir != download_path and current_dir.startswith(download_path): + is_empty = not any(not f.startswith(".") for f in os.listdir(current_dir)) + if is_empty: + logger.warning(f"Removing empty directory: {current_dir}") + os.rmdir(current_dir) + current_dir = os.path.dirname(current_dir) + else: + break + except Exception as e: + logger.error(f"An error occurred during directory cleanup: {e}") + + +def get_audio_quality_string(file_path): + """Return a compact audio quality string for the given file.""" + try: + ext = os.path.splitext(file_path)[1].lower() + + if ext == ".flac": + from mutagen.flac import FLAC + audio = FLAC(file_path) + return f"FLAC {audio.info.bits_per_sample}bit" + + if ext == ".mp3": + from mutagen.mp3 import MP3, BitrateMode + + audio = MP3(file_path) + bitrate_kbps = audio.info.bitrate // 1000 + if audio.info.bitrate_mode == BitrateMode.VBR: + return "MP3-VBR" + return f"MP3-{bitrate_kbps}" + + if ext in (".m4a", ".aac", ".mp4"): + from mutagen.mp4 import MP4 + audio = MP4(file_path) + return f"M4A-{audio.info.bitrate // 1000}" + + if ext == ".ogg": + from mutagen.oggvorbis import OggVorbis + audio = OggVorbis(file_path) + return f"OGG-{audio.info.bitrate // 1000}" + + if ext == ".opus": + from mutagen.oggopus import OggOpus + + audio = OggOpus(file_path) + return f"OPUS-{audio.info.bitrate // 1000}" + + return "" + except Exception as e: + logger.debug(f"Could not determine audio quality for {file_path}: {e}") + return "" + + +def get_quality_tier_from_extension(file_path): + """Classify a file extension into a quality tier.""" + if not file_path: + return ("unknown", 999) + + ext = os.path.splitext(file_path)[1].lower() + quality_tiers = { + "lossless": { + "extensions": [".flac", ".ape", ".wav", ".alac", ".dsf", ".dff", ".aiff", ".aif"], + "tier": 1, + }, + "high_lossy": { + "extensions": [".opus", ".ogg"], + "tier": 2, + }, + "standard_lossy": { + "extensions": [".m4a", ".aac"], + "tier": 3, + }, + "low_lossy": { + "extensions": [".mp3", ".wma"], + "tier": 4, + }, + } + + for tier_name, tier_data in quality_tiers.items(): + if ext in tier_data["extensions"]: + return (tier_name, tier_data["tier"]) + + return ("unknown", 999) + + +def downsample_hires_flac(final_path, context): + """Downsample a hi-res FLAC to 16-bit/44.1kHz if enabled.""" + from mutagen.flac import FLAC + + config_manager = _get_config_manager() + if not config_manager.get("lossy_copy.downsample_hires", False): + return None + + if os.path.splitext(final_path)[1].lower() != ".flac": + return None + + try: + audio = FLAC(final_path) + original_bits = audio.info.bits_per_sample + original_rate = audio.info.sample_rate + except Exception as e: + logger.error(f"[Downsample] Could not read FLAC info: {e}") + return None + + if original_bits <= 16 and original_rate <= 44100: + return None + + logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz -> 16-bit/44100Hz: {os.path.basename(final_path)}") + + ffmpeg_bin = shutil.which("ffmpeg") + if not ffmpeg_bin: + local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg") + if os.path.isfile(local): + ffmpeg_bin = local + else: + logger.warning("[Downsample] ffmpeg not found - skipping hi-res conversion") + return None + + temp_path = final_path + ".tmp.flac" + try: + result = subprocess.run( + [ + ffmpeg_bin, "-i", final_path, + "-sample_fmt", "s16", + "-ar", "44100", + "-map_metadata", "0", + "-compression_level", "8", + "-y", temp_path, + ], + capture_output=True, + text=True, + timeout=300, + ) + + if result.returncode != 0: + logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}") + if os.path.exists(temp_path): + os.remove(temp_path) + return None + + if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0: + logger.warning("[Downsample] Output file missing or empty") + if os.path.exists(temp_path): + os.remove(temp_path) + return None + + verify_audio = FLAC(temp_path) + if verify_audio.info.bits_per_sample != 16: + logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting") + os.remove(temp_path) + return None + + os.replace(temp_path, final_path) + logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}") + + new_quality = "FLAC 16bit" + try: + updated_audio = FLAC(final_path) + updated_audio["QUALITY"] = new_quality + updated_audio.save() + except Exception as tag_err: + logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}") + + old_quality = context.get("_audio_quality", "") + context["_audio_quality"] = new_quality + + if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path): + new_basename = os.path.basename(final_path).replace(old_quality, new_quality) + new_path = os.path.join(os.path.dirname(final_path), new_basename) + try: + os.rename(final_path, new_path) + logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} -> {new_basename}") + for lyrics_ext in (".lrc", ".txt"): + old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext + if os.path.isfile(old_lyrics): + new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext + os.rename(old_lyrics, new_lyrics) + return new_path + except Exception as rename_err: + logger.error(f"[Downsample] Could not rename file: {rename_err}") + + return final_path + except subprocess.TimeoutExpired: + logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}") + if os.path.exists(temp_path): + os.remove(temp_path) + except Exception as e: + logger.error(f"[Downsample] Conversion error: {e}") + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except Exception: + pass + return None + + +def create_lossy_copy(final_path): + """Convert a FLAC file to a lossy copy using the configured codec.""" + from mutagen.flac import FLAC + + config_manager = _get_config_manager() + if not config_manager.get("lossy_copy.enabled", False): + return None + + if os.path.splitext(final_path)[1].lower() != ".flac": + return None + + codec = config_manager.get("lossy_copy.codec", "mp3").lower() + bitrate = config_manager.get("lossy_copy.bitrate", "320") + + if codec == "opus" and int(bitrate) > 256: + bitrate = "256" + + codec_map = { + "mp3": ("libmp3lame", ".mp3", f"MP3-{bitrate}", ["-vn", "-id3v2_version", "3"]), + "opus": ("libopus", ".opus", f"OPUS-{bitrate}", ["-vn", "-map", "0:a", "-vbr", "on"]), + "aac": ("aac", ".m4a", f"AAC-{bitrate}", ["-vn", "-movflags", "+faststart"]), + } + + if codec not in codec_map: + logger.info(f"[Lossy Copy] Unknown codec '{codec}' - skipping conversion") + return None + + ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec] + out_path = os.path.splitext(final_path)[0] + out_ext + + original_quality = get_audio_quality_string(final_path) + if original_quality: + out_basename = os.path.basename(out_path) + if original_quality in out_basename: + out_basename = out_basename.replace(original_quality, quality_label) + out_path = os.path.join(os.path.dirname(out_path), out_basename) + + ffmpeg_bin = shutil.which("ffmpeg") + if not ffmpeg_bin: + local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg") + if os.path.isfile(local): + ffmpeg_bin = local + else: + logger.warning(f"[Lossy Copy] ffmpeg not found - skipping {codec.upper()} conversion") + return None + + try: + logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}") + cmd = [ + ffmpeg_bin, "-i", final_path, + "-codec:a", ffmpeg_codec, + "-b:a", f"{bitrate}k", + "-map_metadata", "0", + ] + extra_args + ["-y", out_path] + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) + + if result.returncode == 0: + logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}") + try: + from mutagen import File as MutagenFile + audio = MutagenFile(out_path) + if audio is not None: + if codec == "mp3": + from mutagen.id3 import TXXX + audio.tags.add(TXXX(encoding=3, desc="QUALITY", text=[quality_label])) + elif codec == "opus": + audio["QUALITY"] = [quality_label] + elif codec == "aac": + from mutagen.mp4 import MP4FreeForm + audio["----:com.apple.iTunes:QUALITY"] = [MP4FreeForm(quality_label.encode("utf-8"))] + audio.save() + except Exception as tag_err: + logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}") + return out_path + + logger.error(f"[Lossy Copy] ffmpeg failed: {result.stderr[:200]}") + if os.path.exists(out_path): + try: + os.remove(out_path) + except Exception: + pass + return None + except subprocess.TimeoutExpired: + logger.warning(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}") + except Exception as e: + logger.error(f"[Lossy Copy] Conversion error: {e}") + return None diff --git a/core/import_filename.py b/core/import_filename.py new file mode 100644 index 00000000..9298750d --- /dev/null +++ b/core/import_filename.py @@ -0,0 +1,73 @@ +"""Filename parsing helpers used by import flows.""" + +from __future__ import annotations + +import os +import re +from typing import Any, Dict + + +_TRACK_PATTERNS = ( + r"^(\d+)\s*[-\.]\s*(.+?)\s*[-–]\s*(.+)$", + r"^(.+?)\s*[-–]\s*(.+)$", + r"^(\d+)\s*[-\.]\s*(.+)$", +) + + +def parse_filename_metadata(filename: str) -> Dict[str, Any]: + """Extract artist/title/album hints from a loose filename.""" + raw_path = str(filename or "") + normalized_path = raw_path.replace("\\", "/") + base_name = os.path.splitext(os.path.basename(normalized_path))[0] + + result: Dict[str, Any] = { + "artist": "", + "title": "", + "album": "", + "track_number": None, + } + + if not base_name: + return result + + for pattern in _TRACK_PATTERNS: + match = re.match(pattern, base_name) + if not match: + continue + + groups = match.groups() + if len(groups) == 3: + try: + result["track_number"] = int(groups[0]) + result["artist"] = result["artist"] or groups[1].strip() + result["title"] = result["title"] or groups[2].strip() + except ValueError: + result["artist"] = result["artist"] or groups[0].strip() + result["title"] = result["title"] or f"{groups[1]} - {groups[2]}".strip() + elif len(groups) == 2: + if groups[0].isdigit(): + try: + result["track_number"] = int(groups[0]) + result["title"] = result["title"] or groups[1].strip() + except ValueError: + pass + else: + result["artist"] = result["artist"] or groups[0].strip() + result["title"] = result["title"] or groups[1].strip() + break + + if not result["title"]: + result["title"] = base_name + + if not result["album"] and "/" in normalized_path: + path_parts = normalized_path.split("/") + for part in reversed(path_parts[:-1]): + if not part or part.startswith("@"): + continue + + cleaned = re.sub(r"^\d+\s*[-\.]\s*", "", part).strip() + if len(cleaned) > 3: + result["album"] = cleaned + break + + return result diff --git a/core/import_guards.py b/core/import_guards.py new file mode 100644 index 00000000..de2a3185 --- /dev/null +++ b/core/import_guards.py @@ -0,0 +1,121 @@ +"""Import post-processing guards and quarantine helpers.""" + +from __future__ import annotations + +import json +import os +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional + +from core.import_context import ( + get_import_clean_artist, + get_import_clean_title, + get_import_context_artist, + get_import_original_search, + get_import_track_info, + normalize_import_context, +) +from core.import_file_ops import safe_move_file +from database.music_database import MusicDatabase +from utils.logging_config import get_logger + + +logger = get_logger("import_guards") + + +def _get_config_manager(): + from config.settings import config_manager + + return config_manager + + +def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None) -> str: + """Move a file to the quarantine folder and write a metadata sidecar.""" + config_manager = _get_config_manager() + download_dir = config_manager.get("soulseek.download_path", "./downloads") + quarantine_dir = Path(download_dir) / "ss_quarantine" + quarantine_dir.mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + original_name = Path(file_path).stem + file_ext = Path(file_path).suffix + + quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined" + quarantine_path = quarantine_dir / quarantine_filename + + safe_move_file(file_path, str(quarantine_path)) + + metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json" + context = normalize_import_context(context) + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + + metadata = { + "original_filename": Path(file_path).name, + "quarantine_reason": reason, + "timestamp": datetime.now().isoformat(), + "expected_track": get_import_clean_title(context, default=original_search.get("title", "Unknown")), + "expected_artist": get_import_clean_artist(context, default=(artist_context.get("name", "") if isinstance(artist_context, dict) else "Unknown")), + "context_key": context.get("context_key", "unknown"), + } + + try: + with open(metadata_path, "w", encoding="utf-8") as f: + json.dump(metadata, f, indent=2, ensure_ascii=False) + except Exception as exc: + logger.warning("Failed to write quarantine metadata: %s", exc) + + logger.warning("File quarantined: %s - Reason: %s", quarantine_path, reason) + + if automation_engine: + try: + ti = context.get("track_info", {}) + artists = ti.get("artists", []) + artist_name = "" + if artists: + first = artists[0] + artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first) + automation_engine.emit( + "download_quarantined", + { + "artist": artist_name, + "title": ti.get("name", ""), + "reason": reason or "Unknown", + }, + ) + except Exception: + pass + + return str(quarantine_path) + + +def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]: + """Return a rejection message if a FLAC file violates the configured bit depth.""" + if not context.get("_audio_quality", "").startswith("FLAC"): + return None + + config_manager = _get_config_manager() + quality_profile = MusicDatabase().get_quality_profile() + flac_config = quality_profile.get("qualities", {}).get("flac", {}) + flac_pref = flac_config.get("bit_depth", "any") + if flac_pref == "any": + return None + + actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "") + if actual_bits == flac_pref: + return None + + flac_fallback = flac_config.get("bit_depth_fallback", True) + downsample_enabled = config_manager.get("lossy_copy.downsample_hires", False) + track_info = context.get("track_info", {}) + track_name = track_info.get("name", os.path.basename(file_path)) + + if flac_fallback or downsample_enabled: + if downsample_enabled: + logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name) + else: + logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name) + return None + + return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit" diff --git a/core/import_paths.py b/core/import_paths.py new file mode 100644 index 00000000..7848425c --- /dev/null +++ b/core/import_paths.py @@ -0,0 +1,757 @@ +"""Shared path and naming helpers for import processing.""" + +from __future__ import annotations + +import json +import logging +import os +import re +import threading +from pathlib import Path +from typing import Any + +from core.import_context import ( + get_import_clean_title, + get_import_context_album, + get_import_original_search, + get_import_source, + get_import_track_info, + normalize_import_context, +) + +logger = logging.getLogger("import_paths") + +_album_cache_lock = threading.Lock() +_album_editions: dict[str, str] = {} +_album_name_cache: dict[str, str] = {} + + +def _get_config_manager(): + try: + from config.settings import config_manager + return config_manager + except Exception: + class _FallbackConfig: + @staticmethod + def get(key, default=None): + return default + + return _FallbackConfig() + + +def _get_itunes_client(): + try: + from core.metadata_service import get_itunes_client + return get_itunes_client() + except Exception: + return None + + +def _get_album_tracks_for_source(source: str, album_id: str): + try: + from core.metadata_service import get_album_tracks_for_source + return get_album_tracks_for_source(source, album_id) + except Exception: + return None + + +def _extract_artist_name(artist_context: Any) -> str: + if not artist_context: + return "" + if isinstance(artist_context, dict): + return str(artist_context.get("name", "") or "").strip() + return str(artist_context).strip() + + +def docker_resolve_path(path_str: str) -> str: + """Resolve Docker-hosted Windows paths into container paths.""" + if os.path.exists("/.dockerenv") and len(path_str) >= 3 and path_str[1] == ":" and path_str[0].isalpha(): + drive_letter = path_str[0].lower() + rest_of_path = path_str[2:].replace("\\", "/") + return f"/host/mnt/{drive_letter}{rest_of_path}" + return path_str + + +def build_simple_download_destination(context, file_path: str): + """Build the destination path for a simple download into Transfer.""" + context = normalize_import_context(context) + search_result = context.get("search_result", {}) or {} + if not isinstance(search_result, dict): + search_result = {} + + transfer_dir = Path(docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer"))) + album_name = None + original_filename = search_result.get("filename", "") + if "/" in original_filename or "\\" in original_filename: + path_parts = original_filename.replace("\\", "/").split("/") + if len(path_parts) >= 2: + album_name = path_parts[-2] + if not album_name: + album_value = search_result.get("album") + if isinstance(album_value, dict): + album_name = album_value.get("name", "") + else: + album_name = album_value + + filename = Path(file_path).name + if album_name and str(album_name).lower() not in {"unknown", "unknown album", ""}: + album_name = sanitize_filename(str(album_name)) + destination_dir = transfer_dir / album_name + else: + album_name = "" + destination_dir = transfer_dir + + destination_dir.mkdir(parents=True, exist_ok=True) + return destination_dir / filename, album_name, filename + + +def sanitize_filename(filename: str) -> str: + """Sanitize filename for file system compatibility.""" + sanitized = re.sub(r'[<>:"/\\|?*]', "_", filename) + sanitized = re.sub(r"\s+", " ", sanitized).strip() + sanitized = sanitized.rstrip(". ") or "_" + if re.match(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)", sanitized, re.IGNORECASE): + sanitized = "_" + sanitized + return sanitized[:200] + + +def sanitize_context_values(context: dict) -> dict: + """Sanitize all string values in a template context for path safety.""" + sanitized = {} + for key, value in context.items(): + if isinstance(value, str) and value: + sanitized[key] = sanitize_filename(value) + else: + sanitized[key] = value + return sanitized + + +def clean_track_title(track_title: str, artist_name: str) -> str: + """Clean up track title by removing artist prefix and other noise.""" + original = (track_title or "").strip() + cleaned = original + cleaned = re.sub(r"^\d{1,2}[\.\s\-]+", "", cleaned) + artist_pattern = re.escape(artist_name or "") + r"\s*-\s*" + cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[A-Za-z0-9\.]+\s*-\s*\d{1,2}\s*-\s*", "", cleaned) + quality_patterns = [ + r"\s*[\[\(][0-9]+\s*kbps[\]\)]\s*", + r"\s*[\[\(]flac[\]\)]\s*", + r"\s*[\[\(]mp3[\]\)]\s*", + ] + for pattern in quality_patterns: + cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[-\s\.]+", "", cleaned) + cleaned = re.sub(r"[-\s\.]+$", "", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + return cleaned if cleaned else original + + +def get_base_album_name(album_name: str) -> str: + """Extract the base album name without edition indicators.""" + base_name = album_name or "" + base_name = re.sub( + r"\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$", + "", + base_name, + flags=re.IGNORECASE, + ) + base_name = re.sub(r"\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$", "", base_name, flags=re.IGNORECASE) + base_name = re.sub( + r"\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", + "", + base_name, + flags=re.IGNORECASE, + ) + return base_name.strip() + + +def detect_deluxe_edition(album_name: str) -> bool: + """Detect if an album name indicates a deluxe/special edition.""" + if not album_name: + return False + + album_lower = album_name.lower() + deluxe_indicators = [ + "deluxe", + "deluxe edition", + "special edition", + "expanded edition", + "extended edition", + "bonus", + "remastered", + "anniversary", + "collectors edition", + "limited edition", + "silver edition", + "gold edition", + "platinum edition", + ] + for indicator in deluxe_indicators: + if indicator in album_lower: + logger.info("Detected deluxe edition: %r contains %r", album_name, indicator) + return True + return False + + +def normalize_base_album_name(base_album: str, artist_name: str) -> str: + """Normalize the base album name to handle case variations and known corrections.""" + normalized_lower = (base_album or "").lower().strip() + known_corrections = { + # Add specific album name corrections here as needed. + } + + for variant, correction in known_corrections.items(): + if normalized_lower == variant.lower(): + logger.info("Album correction applied: %r -> %r", base_album, correction) + return correction + + normalized = base_album or "" + normalized = re.sub(r"\s*&\s*", " & ", normalized) + normalized = re.sub(r"\s+", " ", normalized) + normalized = normalized.strip() + logger.info("Album variant normalization: %r -> %r", base_album, normalized) + return normalized + + +def clean_album_title(album_title: str, artist_name: str) -> str: + """Clean up album title by removing common prefixes, suffixes, and artist redundancy.""" + original = (album_title or "").strip() + cleaned = original + logger.info("Album Title Cleaning: %r (artist: %r)", original, artist_name) + + cleaned = re.sub(r"^Album\s*-\s*", "", cleaned, flags=re.IGNORECASE) + artist_pattern = re.escape(artist_name or "") + r"\s*-\s*" + cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*[\[\(]\d{4}[\]\)]\s*", " ", cleaned) + + quality_patterns = [ + r"\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*", + r"\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*", + r"\s*[\[\(].*?flac.*?[\]\)]\s*", + r"\s*[\[\(].*?mp3.*?[\]\)]\s*", + r"\s*[\[\(].*?itunes.*?[\]\)]\s*", + r"\s*[\[\(].*?web.*?[\]\)]\s*", + r"\s*[\[\(].*?cd.*?[\]\)]\s*", + ] + for pattern in quality_patterns: + cleaned = re.sub(pattern, " ", cleaned, flags=re.IGNORECASE) + + cleaned = re.sub(r"\s*[\[\(][^\]\)]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*[\[\(][^\]\)]*\bedition\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[-\s\.]+", "", cleaned) + cleaned = re.sub(r"[-\s\.]+$", "", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + return cleaned if cleaned else original + + +def resolve_album_group(artist_context: dict, album_info: dict, original_album: str = None) -> str: + """Smart album grouping: upgrade to deluxe if any track is deluxe.""" + try: + with _album_cache_lock: + artist_name = _extract_artist_name(artist_context) + detected_album = (album_info or {}).get("album_name", "") + + if detected_album: + base_album = get_base_album_name(detected_album) + elif original_album: + cleaned_original = clean_album_title(original_album, artist_name) + base_album = get_base_album_name(cleaned_original) + else: + base_album = get_base_album_name(detected_album) + + base_album = normalize_base_album_name(base_album, artist_name) + album_key = f"{artist_name}::{base_album}" + is_deluxe_track = False + if detected_album: + is_deluxe_track = detect_deluxe_edition(detected_album) + elif original_album: + is_deluxe_track = detect_deluxe_edition(original_album) + + if album_key in _album_name_cache: + cached_name = _album_name_cache[album_key] + current_edition = _album_editions.get(album_key, "standard") + if is_deluxe_track and current_edition == "standard": + final_album_name = f"{base_album} (Deluxe Edition)" + _album_editions[album_key] = "deluxe" + _album_name_cache[album_key] = final_album_name + logger.info("Album cache upgrade: %r -> %r", album_key, final_album_name) + return final_album_name + logger.info("Using cached album name for %r: %r", album_key, cached_name) + return cached_name + + logger.info("Album grouping - Key: %r, Detected: %r", album_key, detected_album) + + current_edition = _album_editions.get(album_key, "standard") + if is_deluxe_track and current_edition == "standard": + logger.info("UPGRADE: Album %r upgraded from standard to deluxe!", base_album) + _album_editions[album_key] = "deluxe" + current_edition = "deluxe" + + if current_edition == "deluxe": + final_album_name = f"{base_album} (Deluxe Edition)" + else: + final_album_name = base_album + + _album_name_cache[album_key] = final_album_name + + logger.info("Album resolution: %r -> %r (edition: %s)", detected_album, final_album_name, current_edition) + return final_album_name + except Exception as e: + logger.error("Error resolving album group: %s", e) + album_name = (album_info or {}).get("album_name", "Unknown Album") + return album_name + + +def get_album_type_display(raw_type, track_count) -> str: + """Return the display form of an album's type for the $albumtype template variable.""" + raw = (raw_type or "").strip().lower() + try: + tc = int(track_count or 0) + except (TypeError, ValueError): + tc = 0 + + if raw in ("compilation", "compile"): + return "Compilation" + if raw == "album": + return "Album" + if raw in ("single", "ep"): + if tc <= 3: + return "Single" + if tc <= 6: + return "EP" + return "Album" + + if tc <= 0: + return "Album" + if tc <= 3: + return "Single" + if tc <= 6: + return "EP" + return "Album" + + +def _replace_template_variables(template: str, context: dict) -> str: + clean_context = sanitize_context_values(context) + result = template + + album_artist_value = clean_context.get("albumartist", clean_context.get("artist", "Unknown Artist")) + collab_mode = _get_config_manager().get("file_organization.collab_artist_mode", "first") + if collab_mode == "first" and album_artist_value: + artists_list = context.get("_artists_list") + if artists_list and len(artists_list) > 1: + first = artists_list[0] + album_artist_value = first.get("name", first) if isinstance(first, dict) else str(first) + elif artists_list and len(artists_list) == 1: + itunes_artist_id = context.get("_itunes_artist_id") + if itunes_artist_id and ("," in album_artist_value or " & " in album_artist_value): + try: + resolved_client = _get_itunes_client() + if resolved_client and hasattr(resolved_client, "resolve_primary_artist"): + resolved = resolved_client.resolve_primary_artist(itunes_artist_id) + if resolved and resolved != album_artist_value: + album_artist_value = resolved + except Exception: + pass + + bracket_map = { + "albumartist": album_artist_value, + "albumtype": clean_context.get("albumtype", "Album"), + "playlist": clean_context.get("playlist_name", ""), + "artistletter": (clean_context.get("artist", "U") or "U")[0].upper(), + "artist": clean_context.get("artist", "Unknown Artist"), + "album": clean_context.get("album", "Unknown Album"), + "title": clean_context.get("title", "Unknown Track"), + "track": f"{_coerce_int(clean_context.get('track_number', 1), 1):02d}", + "disc": str(_coerce_int(clean_context.get("disc_number", 1), 1)), + "discnum": str(_coerce_int(clean_context.get("disc_number", 1), 1)), + "year": str(clean_context.get("year", "")), + "quality": clean_context.get("quality", ""), + } + for var_name, val in bracket_map.items(): + result = result.replace("${" + var_name + "}", val) + + result = result.replace("$albumartist", album_artist_value) + result = result.replace("$albumtype", clean_context.get("albumtype", "Album")) + result = result.replace("$playlist", clean_context.get("playlist_name", "")) + result = result.replace("$artistletter", (clean_context.get("artist", "U") or "U")[0].upper()) + result = result.replace("$artist", clean_context.get("artist", "Unknown Artist")) + result = result.replace("$album", clean_context.get("album", "Unknown Album")) + result = result.replace("$title", clean_context.get("title", "Unknown Track")) + result = result.replace("$track", f"{clean_context.get('track_number', 1):02d}") + result = result.replace("$year", str(clean_context.get("year", ""))) + + result = re.sub(r"\s+", " ", result) + result = re.sub(r"\s*-\s*-\s*", " - ", result) + result = result.strip() + return result + + +def apply_path_template(template: str, context: dict) -> str: + """Apply a template to build a path string.""" + return _replace_template_variables(template, context) + + +def get_file_path_from_template_raw(template: str, context: dict) -> tuple[str, str]: + """Build file path using a user-provided template string directly.""" + full_path = apply_path_template(template, context) + + quality_value = context.get("quality", "") + disc_number = _coerce_int(context.get("disc_number", 1), 1) + disc_value = f"{disc_number:02d}" + disc_value_raw = str(disc_number) + + path_parts = full_path.split("/") + if len(path_parts) > 1: + folder_parts = path_parts[:-1] + filename_base = path_parts[-1] + + cleaned_folders = [] + for part in folder_parts: + part = part.replace("$quality", "") + part = part.replace("$discnum", "") + part = part.replace("$disc", "") + part = re.sub(r"\s*\[\s*\]", "", part) + part = re.sub(r"\s*\(\s*\)", "", part) + part = re.sub(r"\s*\{\s*\}", "", part) + part = re.sub(r"\s*-\s*$", "", part) + part = re.sub(r"^\s*-\s*", "", part) + part = re.sub(r"\s+", " ", part).strip() + if part: + cleaned_folders.append(part) + + filename_base = filename_base.replace("$quality", quality_value) + filename_base = filename_base.replace("$discnum", disc_value_raw) + filename_base = filename_base.replace("$disc", disc_value) + filename_base = re.sub(r"\s*\[\s*\]", "", filename_base) + filename_base = re.sub(r"\s*\(\s*\)", "", filename_base) + filename_base = re.sub(r"\s*\{\s*\}", "", filename_base) + filename_base = re.sub(r"\s*-\s*$", "", filename_base) + filename_base = re.sub(r"\s+", " ", filename_base).strip() + + sanitized_folders = [sanitize_filename(part) for part in cleaned_folders] + folder_path = os.path.join(*sanitized_folders) if sanitized_folders else "" + return folder_path, sanitize_filename(filename_base) + + full_path = full_path.replace("$quality", quality_value) + full_path = full_path.replace("$discnum", disc_value_raw) + full_path = full_path.replace("$disc", disc_value) + full_path = re.sub(r"\s*\[\s*\]", "", full_path) + full_path = re.sub(r"\s*\(\s*\)", "", full_path) + full_path = re.sub(r"\s*\{\s*\}", "", full_path) + full_path = re.sub(r"\s*-\s*$", "", full_path) + full_path = re.sub(r"\s+", " ", full_path).strip() + return "", sanitize_filename(full_path) + + +def get_file_path_from_template(context: dict, template_type: str = "album_path") -> tuple[str, str]: + """Build complete file path using configured templates.""" + if not _get_config_manager().get("file_organization.enabled", True): + return None, None + + templates = _get_config_manager().get("file_organization.templates", {}) + template = templates.get(template_type) + if not template: + default_templates = { + "album_path": "$albumartist/$albumartist - $album/$track - $title", + "single_path": "$artist/$artist - $title/$title", + "compilation_path": "Compilations/$album/$track - $artist - $title", + "playlist_path": "$playlist/$artist - $title", + } + template = default_templates.get(template_type, "$artist/$album/$track - $title") + + full_path = apply_path_template(template, context) + + path_parts = full_path.split("/") + quality_value = context.get("quality", "") + disc_number = _coerce_int(context.get("disc_number", 1), 1) + disc_value = f"{disc_number:02d}" + disc_value_raw = str(disc_number) + + if len(path_parts) > 1: + folder_parts = path_parts[:-1] + filename_base = path_parts[-1] + + cleaned_folders = [] + for part in folder_parts: + part = part.replace("$quality", "") + part = part.replace("$discnum", "") + part = part.replace("$disc", "") + part = re.sub(r"\s*\[\s*\]", "", part) + part = re.sub(r"\s*\(\s*\)", "", part) + part = re.sub(r"\s*\{\s*\}", "", part) + part = re.sub(r"\s*-\s*$", "", part) + part = re.sub(r"^\s*-\s*", "", part) + part = re.sub(r"\s+", " ", part).strip() + if part: + cleaned_folders.append(part) + + filename_base = filename_base.replace("$quality", quality_value) + filename_base = filename_base.replace("$discnum", disc_value_raw) + filename_base = filename_base.replace("$disc", disc_value) + filename_base = re.sub(r"\s*\[\s*\]", "", filename_base) + filename_base = re.sub(r"\s*\(\s*\)", "", filename_base) + filename_base = re.sub(r"\s*\{\s*\}", "", filename_base) + filename_base = re.sub(r"\s*-\s*$", "", filename_base) + filename_base = re.sub(r"\s+", " ", filename_base).strip() + + sanitized_folders = [sanitize_filename(part) for part in cleaned_folders] + folder_path = os.path.join(*sanitized_folders) if sanitized_folders else "" + filename = sanitize_filename(filename_base) + return folder_path, filename + + full_path = full_path.replace("$quality", quality_value) + full_path = full_path.replace("$discnum", disc_value_raw) + full_path = full_path.replace("$disc", disc_value) + full_path = re.sub(r"\s*\[\s*\]", "", full_path) + full_path = re.sub(r"\s*\(\s*\)", "", full_path) + full_path = re.sub(r"\s*\{\s*\}", "", full_path) + full_path = re.sub(r"\s*-\s*$", "", full_path) + full_path = re.sub(r"\s+", " ", full_path).strip() + return "", sanitize_filename(full_path) + + +def _max_disc_number(album_tracks: Any) -> int: + items = [] + if isinstance(album_tracks, dict): + items = album_tracks.get("items") or album_tracks.get("tracks") or [] + elif isinstance(album_tracks, list): + items = album_tracks + + max_disc = 1 + for track in items: + if not isinstance(track, dict): + continue + try: + disc_number = int(track.get("disc_number", 1) or 1) + except (TypeError, ValueError): + disc_number = 1 + if disc_number > max_disc: + max_disc = disc_number + return max_disc + + +def _coerce_int(value: Any, default: int = 1) -> int: + try: + coerced = int(value) + except (TypeError, ValueError): + return default + return coerced if coerced > 0 else default + + +def build_final_path_for_track(context, artist_context, album_info, file_ext): + """Shared path builder used by both post-processing and verification.""" + transfer_dir = docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer")) + context = normalize_import_context(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + album_context = get_import_context_album(context) + source = get_import_source(context) + playlist_folder_mode = track_info.get("_playlist_folder_mode", False) + artist_name = _extract_artist_name(artist_context) + + source_info = track_info.get("source_info") or {} + if isinstance(source_info, str): + try: + source_info = json.loads(source_info) + except (json.JSONDecodeError, TypeError): + source_info = {} + if source_info.get("enhance") and source_info.get("original_file_path"): + original_path = source_info["original_file_path"] + original_dir = os.path.dirname(original_path) + original_stem = os.path.splitext(os.path.basename(original_path))[0] + final_path = os.path.join(original_dir, original_stem + file_ext) + os.makedirs(original_dir, exist_ok=True) + logger.info("[Enhance] Using original file location: %s", final_path) + return final_path, True + + year = "" + if album_context and album_context.get("release_date"): + release_date = album_context["release_date"] + if release_date and len(release_date) >= 4: + year = release_date[:4] + + raw_album_type = "" + if album_context: + raw_album_type = album_context.get("album_type", "") or "" + total_tracks = (album_context.get("total_tracks", 0) or 0) if album_context else 0 + album_type_display = get_album_type_display(raw_album_type, total_tracks) + + if playlist_folder_mode: + playlist_name = track_info.get("_playlist_name", "Unknown Playlist") + track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) + _artists = original_search.get("artists") or track_info.get("artists") or [] + + template_context = { + "artist": artist_name, + "albumartist": artist_name, + "album": track_name, + "title": track_name, + "playlist_name": playlist_name, + "track_number": 1, + "disc_number": 1, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _artists, + "_itunes_artist_id": str(artist_context.get("id", "")) if isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source == "itunes" else None, + } + + folder_path, filename_base = get_file_path_from_template(template_context, "playlist_path") + if folder_path and filename_base: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + return final_path, True + + playlist_name_sanitized = sanitize_filename(playlist_name) + playlist_dir = os.path.join(transfer_dir, playlist_name_sanitized) + os.makedirs(playlist_dir, exist_ok=True) + artist_name_sanitized = sanitize_filename(template_context["artist"]) + track_name_sanitized = sanitize_filename(track_name) + new_filename = f"{artist_name_sanitized} - {track_name_sanitized}{file_ext}" + return os.path.join(playlist_dir, new_filename), True + + if album_info and album_info.get("is_album"): + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + track_number = _coerce_int(album_info.get("track_number", 1), 1) + disc_number = _coerce_int(album_info.get("disc_number", 1), 1) + _artists = original_search.get("artists") or track_info.get("artists") or [] + _album_ctx = album_context + _itunes_aid = None + _is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer") + if _is_itunes and isinstance(artist_context, dict): + _aid = artist_context.get("id", "") + if str(_aid).isdigit(): + _itunes_aid = str(_aid) + if not _itunes_aid and _album_ctx: + _ext = _album_ctx.get("external_urls", {}) + if isinstance(_ext, dict) and _ext.get("itunes_artist_id"): + _itunes_aid = _ext["itunes_artist_id"] + + _artist_name = artist_name + _album_artist_name = _artist_name + _album_artists_for_collab = None + _explicit_artist_ctx = track_info.get("_explicit_artist_context") if isinstance(track_info, dict) else None + if isinstance(_explicit_artist_ctx, dict) and _explicit_artist_ctx.get("name"): + _album_artist_name = _explicit_artist_ctx["name"] + _album_artists_for_collab = [_explicit_artist_ctx] + elif isinstance(_explicit_artist_ctx, str) and _explicit_artist_ctx: + _album_artist_name = _explicit_artist_ctx + _album_artists_for_collab = [{"name": _explicit_artist_ctx}] + else: + _sa_artists = _album_ctx.get("artists", []) if _album_ctx else [] + if _sa_artists: + _first_sa = _sa_artists[0] + if isinstance(_first_sa, dict) and _first_sa.get("name"): + _album_artist_name = _first_sa["name"] + elif isinstance(_first_sa, str) and _first_sa: + _album_artist_name = _first_sa + _album_artists_for_collab = _sa_artists + + template_context = { + "artist": _artist_name, + "albumartist": _album_artist_name, + "album": album_info["album_name"], + "title": clean_track_name, + "track_number": track_number, + "disc_number": disc_number, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _album_artists_for_collab if _album_artists_for_collab else _artists, + "_itunes_artist_id": _itunes_aid, + } + total_discs = _coerce_int(album_context.get("total_discs", 1) if album_context else 1, 1) + + if total_discs <= 1 and album_context and album_context.get("id"): + if disc_number > 1: + total_discs = disc_number + else: + try: + _album_tracks = _get_album_tracks_for_source(source, str(album_context["id"])) + if _album_tracks: + total_discs = _max_disc_number(_album_tracks) + if total_discs > 1: + album_context["total_discs"] = total_discs + logger.info( + "[Multi-Disc] Resolved %s discs for single-track download of %r", + total_discs, + album_context.get("name"), + ) + except Exception as _disc_err: + logger.warning("[Multi-Disc] Could not resolve total_discs: %s", _disc_err) + + album_template = _get_config_manager().get("file_organization.templates.album_path", "") + user_controls_disc = "$disc" in album_template + disc_label = _get_config_manager().get("file_organization.disc_label", "Disc") + + folder_path, filename_base = get_file_path_from_template(template_context, "album_path") + if folder_path and filename_base: + if total_discs > 1 and not user_controls_disc: + disc_folder = f"{disc_label} {disc_number}" + final_path = os.path.join(transfer_dir, folder_path, disc_folder, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True) + else: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + return final_path, True + + artist_name_sanitized = sanitize_filename(template_context["albumartist"]) + album_name_sanitized = sanitize_filename(album_info["album_name"]) + artist_dir = os.path.join(transfer_dir, artist_name_sanitized) + album_folder_name = f"{artist_name_sanitized} - {album_name_sanitized}" + album_dir = os.path.join(artist_dir, album_folder_name) + if total_discs > 1: + album_dir = os.path.join(album_dir, f"{disc_label} {disc_number}") + os.makedirs(album_dir, exist_ok=True) + final_track_name_sanitized = sanitize_filename(clean_track_name) + new_filename = f"{track_number:02d} - {final_track_name_sanitized}{file_ext}" + return os.path.join(album_dir, new_filename), True + + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + _artists = original_search.get("artists") or track_info.get("artists") or [] + _album_ctx = album_context + _itunes_aid = None + _is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer") + if _is_itunes and isinstance(artist_context, dict): + _aid = artist_context.get("id", "") + if str(_aid).isdigit(): + _itunes_aid = str(_aid) + if not _itunes_aid and _album_ctx: + _ext = _album_ctx.get("external_urls", {}) + if isinstance(_ext, dict) and _ext.get("itunes_artist_id"): + _itunes_aid = _ext["itunes_artist_id"] + + template_context = { + "artist": artist_name, + "albumartist": artist_name, + "album": album_info.get("album_name", clean_track_name) if album_info else clean_track_name, + "title": clean_track_name, + "track_number": 1, + "disc_number": 1, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _artists, + "_itunes_artist_id": _itunes_aid, + } + + folder_path, filename_base = get_file_path_from_template(template_context, "single_path") + if filename_base: + if folder_path: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + else: + final_path = os.path.join(transfer_dir, filename_base + file_ext) + os.makedirs(transfer_dir, exist_ok=True) + return final_path, True + + artist_name_sanitized = sanitize_filename(template_context["artist"]) + final_track_name_sanitized = sanitize_filename(clean_track_name) + artist_dir = os.path.join(transfer_dir, artist_name_sanitized) + single_folder_name = f"{artist_name_sanitized} - {final_track_name_sanitized}" + single_dir = os.path.join(artist_dir, single_folder_name) + os.makedirs(single_dir, exist_ok=True) + new_filename = f"{final_track_name_sanitized}{file_ext}" + return os.path.join(single_dir, new_filename), True diff --git a/core/import_pipeline.py b/core/import_pipeline.py new file mode 100644 index 00000000..ede3ae18 --- /dev/null +++ b/core/import_pipeline.py @@ -0,0 +1,925 @@ +"""Import/post-processing pipeline for downloads and imported files.""" + +from __future__ import annotations + +import json +import os +import threading +import time + +from config.settings import config_manager +from core.import_file_ops import ( + cleanup_empty_directories, + create_lossy_copy, + downsample_hires_flac, + extract_track_number_from_filename, + get_audio_quality_string, + get_quality_tier_from_extension, + safe_move_file, +) +from core.import_context import ( + build_import_album_info, + extract_artist_name, + get_import_clean_artist, + get_import_clean_title, + get_import_context_artist, + get_import_has_clean_metadata, + get_import_original_search, + get_import_source, + get_import_track_info, + normalize_import_context, +) +from core.import_guards import check_flac_bit_depth, move_to_quarantine +from core.import_side_effects import ( + check_and_remove_from_wishlist, + emit_track_downloaded, + record_download_provenance, + record_library_history_download, + record_retag_download, + record_soulsync_library_entry, +) +from core.import_runtime_state import ( + add_activity_item, + detect_album_info_web, + download_batches, + download_tasks, + matched_context_lock, + matched_downloads_context, + mark_task_completed as _mark_task_completed, + _post_process_locks, + _post_process_locks_lock, + _processed_download_ids, + tasks_lock, +) +from core.metadata_enrichment import ( + download_cover_art, + enhance_file_metadata, + generate_lrc_file, + wipe_source_tags, +) +from core.import_paths import ( + build_final_path_for_track, + build_simple_download_destination, + docker_resolve_path, + resolve_album_group, +) +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("import_pipeline") +pp_logger = get_logger("post_processing") + + +def post_process_matched_download(context_key, context, file_path, runtime): + on_download_completed = getattr(runtime, "on_download_completed", None) + automation_engine = getattr(runtime, "automation_engine", None) + web_scan_manager = getattr(runtime, "web_scan_manager", None) + repair_worker = getattr(runtime, "repair_worker", None) + + def _notify_download_completed(batch_id, task_id, success=True): + if on_download_completed: + on_download_completed(batch_id, task_id, success=success) + + with _post_process_locks_lock: + if context_key not in _post_process_locks: + _post_process_locks[context_key] = threading.Lock() + file_lock = _post_process_locks[context_key] + + file_lock.acquire() + try: + if not os.path.exists(file_path): + existing_final = context.get('_final_processed_path') + if existing_final and os.path.exists(existing_final): + logger.info( + f"[Race Guard] Source gone but destination exists — already processed by another thread: " + f"{os.path.basename(existing_final)}" + ) + return + logger.error( + f"[Race Guard] Source file gone and no known destination — marking as failed: " + f"{os.path.basename(file_path)}" + ) + context['_race_guard_failed'] = True + return + + _basename = os.path.basename(file_path) + _prev_size = -1 + for _stability_check in range(5): + try: + _cur_size = os.path.getsize(file_path) + except OSError: + _cur_size = -1 + if _cur_size == _prev_size and _cur_size > 0: + break + _prev_size = _cur_size + if _stability_check == 0: + logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)") + time.sleep(1.5) + else: + logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)") + + _skip_acoustid = False + try: + from core.acoustid_verification import AcoustIDVerification, VerificationResult + + verifier = AcoustIDVerification() + available, available_reason = verifier.quick_check_available() + if available and not _skip_acoustid: + context = normalize_import_context(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + + expected_track = get_import_clean_title(context, default=original_search.get('title', '')) + expected_artist = '' + track_artists = track_info.get('artists', []) + if track_artists: + first = track_artists[0] + if isinstance(first, dict): + expected_artist = first.get('name', '') + elif isinstance(first, str): + expected_artist = first + if not expected_artist: + expected_artist = extract_artist_name(artist_context) or get_import_clean_artist(context, default='') + + if expected_track and expected_artist: + logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'") + verification_result, verification_msg = verifier.verify_audio_file( + file_path, + expected_track, + expected_artist, + context, + ) + logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}") + context['_acoustid_result'] = verification_result.value + + if verification_result == VerificationResult.FAIL: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + verification_msg, + automation_engine, + ) + logger.error(f"File quarantined due to verification failure: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}") + logger.error(f"Quarantine failed, deleting wrong file: {file_path}") + try: + os.remove(file_path) + except Exception as del_error: + logger.error(f"Could not delete wrong file either: {del_error}") + + context['_acoustid_quarantined'] = True + context['_acoustid_failure_msg'] = verification_msg + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"AcoustID verification failed: {verification_msg}" + ) + + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + else: + logger.warning("AcoustID verification skipped: missing track/artist info") + context['_acoustid_result'] = 'skip' + else: + logger.info(f"ℹ️ AcoustID verification not available: {available_reason}") + context['_acoustid_result'] = 'disabled' + except Exception as verify_error: + logger.error(f"AcoustID verification error (continuing normally): {verify_error}") + context['_acoustid_result'] = 'error' + + search_result = context.get('search_result', {}) or {} + if not isinstance(search_result, dict): + search_result = {} + is_simple_download = search_result.get('is_simple_download', False) + if is_simple_download: + logger.info(f"Processing simple download (no metadata enhancement): {file_path}") + + destination, album_name, filename = build_simple_download_destination(context, file_path) + if album_name: + logger.info(f"Moving to album folder: {album_name}") + else: + logger.info("Moving to Transfer root (single track)") + + safe_move_file(file_path, destination) + logger.info(f"Moved simple download to: {destination}") + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + if web_scan_manager: + threading.Thread( + target=lambda: web_scan_manager.request_scan("Simple download completed"), + daemon=True, + ).start() + + activity_target = f"{album_name}/{filename}" if album_name else filename + add_activity_item("", "Download Complete", activity_target, "Now") + logger.info(f"Simple download post-processing complete: {activity_target}") + context['_simple_download_completed'] = True + context['_final_path'] = str(destination) + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + return + + logger.info(f"Starting robust post-processing for: {context_key}") + + context = normalize_import_context(context) + artist_context = get_import_context_artist(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + has_clean_metadata = get_import_has_clean_metadata(context) + + if not artist_context: + logger.error("Post-processing failed: Missing artist context.") + return + + _junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'} + _artist_name = (artist_context.get('name', '') if isinstance(artist_context, dict) else '').strip() + if _artist_name.lower() in _junk_artist_names: + logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve") + _resolved = False + track_info_guard = track_info or {} + original_search_guard = original_search or {} + + _ti_artists = track_info_guard.get('artists', []) + if isinstance(_ti_artists, list) and _ti_artists: + _first = _ti_artists[0] + _name = _first.get('name', '') if isinstance(_first, dict) else str(_first) + if _name and _name.strip().lower() not in _junk_artist_names: + artist_context['name'] = _name.strip() + logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'") + _resolved = True + + if not _resolved: + _os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or '' + if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names: + artist_context['name'] = _os_artist.strip() + logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'") + _resolved = True + + if not _resolved: + _track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or '' + if _track_id: + try: + from core.metadata_service import get_client_for_source, get_primary_source + + _guard_source = get_import_source(context) or get_primary_source() + _fb_client = get_client_for_source(_guard_source) or get_client_for_source(get_primary_source()) + if hasattr(_fb_client, 'get_track_details'): + _details = _fb_client.get_track_details(str(_track_id)) + if _details and isinstance(_details, dict): + _d_artists = _details.get('artists', []) + if isinstance(_d_artists, list) and _d_artists: + _d_first = _d_artists[0] + _d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first) + if _d_name and _d_name.strip().lower() not in _junk_artist_names: + artist_context['name'] = _d_name.strip() + logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'") + _resolved = True + except Exception as _guard_err: + logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}") + + if not _resolved: + logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'") + + context['artist'] = artist_context + + playlist_folder_mode = track_info.get("_playlist_folder_mode", False) + logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}") + logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}") + if track_info: + logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}") + + if playlist_folder_mode: + playlist_name = track_info.get("_playlist_name", "Unknown Playlist") + logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}") + + file_ext = os.path.splitext(file_path)[1] + final_path, _ = build_final_path_for_track(context, artist_context, None, file_ext) + logger.info(f"Playlist mode final path: '{final_path}'") + + if not os.path.exists(file_path): + if os.path.exists(final_path): + logger.info( + f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: " + f"{os.path.basename(final_path)}" + ) + context['_final_processed_path'] = final_path + return + pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}") + raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}") + + context['_audio_quality'] = get_audio_quality_string(file_path) + if context['_audio_quality']: + logger.info(f"Audio quality detected: {context['_audio_quality']}") + + rejection_reason = check_flac_bit_depth(file_path, context) + if rejection_reason: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + rejection_reason, + automation_engine, + ) + logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") + try: + os.remove(file_path) + except Exception: + pass + + context['_bitdepth_rejected'] = True + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}" + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + + try: + logger.warning( + f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' " + f"(id: {artist_context.get('id', 'MISSING')})" + ) + enhance_file_metadata(file_path, context, artist_context, None) + except Exception as meta_err: + import traceback + pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") + wipe_source_tags(file_path) + + logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") + safe_move_file(file_path, final_path) + context['_final_processed_path'] = final_path + + if config_manager.get('post_processing.replaygain_enabled', False): + try: + from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF + if _rg_ffmpeg_ok(): + lufs, peak_dbfs = _rg_analyze(final_path) + gain_db = _RG_REF - lufs + _rg_write(final_path, gain_db, peak_dbfs) + pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}") + except Exception as rg_err: + pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") + + downsampled_path = downsample_hires_flac(final_path, context) + if downsampled_path: + final_path = downsampled_path + context['_final_processed_path'] = final_path + + blasphemy_path = create_lossy_copy(final_path) + if blasphemy_path: + context['_final_processed_path'] = blasphemy_path + + downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + cleanup_empty_directories(downloads_path, file_path) + + logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}") + + try: + check_and_remove_from_wishlist(context) + except Exception as wishlist_error: + logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}") + + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id and batch_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['stream_processed'] = True + download_tasks[task_id]['status'] = 'completed' + logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed") + _notify_download_completed(batch_id, task_id, success=True) + return + + is_album_download = bool(context.get("is_album_download", False)) + album_info = build_import_album_info(context, force_album=is_album_download) + + if is_album_download: + if has_clean_metadata: + logger.info("Album context with clean metadata found - using normalized album info") + else: + logger.warning("Album context found without clean metadata - using normalized album info") + elif not album_info.get('is_album'): + logger.info("Single track download - attempting album detection") + detected_album_info = detect_album_info_web(context, artist_context) + if detected_album_info: + album_info = detected_album_info + + if album_info and album_info['is_album'] and not is_album_download: + logger.info( + "SMART ALBUM GROUPING for track=%r original_album=%r", + album_info.get('clean_track_name', 'Unknown'), + album_info.get('album_name', 'None'), + ) + original_album = original_search.get("album") if original_search.get("album") else None + consistent_album_name = resolve_album_group(artist_context, album_info, original_album) + album_info['album_name'] = consistent_album_name + logger.info("Album grouping complete: final_album=%r", consistent_album_name) + elif album_info and album_info['is_album'] and is_album_download: + logger.info( + "EXPLICIT ALBUM DOWNLOAD - preserving album name=%r; skipping smart grouping", + album_info.get('album_name', 'None'), + ) + + context['_audio_quality'] = get_audio_quality_string(file_path) + if context['_audio_quality']: + logger.info(f"Audio quality detected: {context['_audio_quality']}") + + rejection_reason = check_flac_bit_depth(file_path, context) + if rejection_reason: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + rejection_reason, + automation_engine, + ) + logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") + try: + os.remove(file_path) + except Exception: + pass + + context['_bitdepth_rejected'] = True + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}" + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + + file_ext = os.path.splitext(file_path)[1] + clean_track_name = get_import_clean_title( + context, + album_info=album_info, + default=original_search.get('title', 'Unknown Track'), + ) + track_number = album_info.get('track_number', 1) + logger.debug( + "Final track_number processing: source=%s album_info_track_number=%s track_number=%s", + album_info.get('source', 'unknown'), + album_info.get('track_number', 'NOT_FOUND'), + track_number, + ) + if track_number is None: + track_number = extract_track_number_from_filename(file_path) + logger.info( + "Track number was None; extracted from filename=%r -> %s", + os.path.basename(file_path), + track_number, + ) + if not isinstance(track_number, int) or track_number < 1: + logger.error(f"Invalid track number ({track_number}), defaulting to 1") + track_number = 1 + + logger.debug(f"FINAL track_number used for filename: {track_number}") + album_info['track_number'] = track_number + album_info['clean_track_name'] = clean_track_name + logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata") + + final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext) + logger.info(f"Resolved path: '{final_path}'") + context['_final_processed_path'] = final_path + + try: + logger.warning(f"[Metadata Input] artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})") + if album_info: + logger.warning( + f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', " + f"track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, " + f"source: {album_info.get('source', 'unknown')}" + ) + else: + logger.info("[Metadata Input] album_info: None (single track)") + enhance_file_metadata(file_path, context, artist_context, album_info) + except Exception as meta_err: + import traceback + pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") + wipe_source_tags(file_path) + + _enhance_source_info = context.get('track_info', {}).get('source_info') or {} + if isinstance(_enhance_source_info, str): + try: + _enhance_source_info = json.loads(_enhance_source_info) + except (json.JSONDecodeError, TypeError): + _enhance_source_info = {} + is_enhance_download = _enhance_source_info.get('enhance', False) + + logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") + if os.path.exists(final_path): + if not os.path.exists(file_path): + logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}") + return + try: + from mutagen import File as MutagenFile + existing_file = MutagenFile(final_path) + has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2 + if has_metadata and not is_enhance_download: + _replace_lower = config_manager.get('import.replace_lower_quality', False) + if _replace_lower: + _existing_tier = get_quality_tier_from_extension(final_path) + _incoming_tier = get_quality_tier_from_extension(file_path) + if _incoming_tier[1] < _existing_tier[1]: + logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except Exception as e: + logger.error(f"[Quality Replace] Could not remove existing file: {e}") + else: + logger.info( + f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: " + f"{os.path.basename(final_path)}" + ) + try: + os.remove(file_path) + except FileNotFoundError: + pass + except Exception as e: + logger.error(f"[Protection] Error removing redundant file: {e}") + return + else: + logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}") + logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}") + try: + os.remove(file_path) + except FileNotFoundError: + logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}") + except Exception as e: + logger.error(f"[Protection] Error removing redundant file: {e}") + return + elif is_enhance_download: + logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except Exception as e: + logger.error(f"[Enhance] Could not remove existing file for replacement: {e}") + else: + logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except FileNotFoundError: + pass + except Exception as check_error: + logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}") + try: + if os.path.exists(final_path): + os.remove(final_path) + except Exception as e: + logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}") + + if not os.path.exists(file_path): + if os.path.exists(final_path): + logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}") + download_cover_art(album_info, os.path.dirname(final_path), context) + generate_lrc_file(final_path, context, artist_context, album_info) + return + expected_dir = os.path.dirname(final_path) + expected_stem = os.path.splitext(os.path.basename(final_path))[0] + expected_ext = os.path.splitext(final_path)[1] + found_variant = None + check_exts = {expected_ext} + if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False): + _lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'} + _lossy_codec = config_manager.get('lossy_copy.codec', 'mp3') + check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3')) + if os.path.exists(expected_dir): + for f in os.listdir(expected_dir): + f_ext = os.path.splitext(f)[1].lower() + if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem): + found_variant = os.path.join(expected_dir, f) + break + if found_variant: + logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}") + context['_final_processed_path'] = found_variant + download_cover_art(album_info, expected_dir, context) + generate_lrc_file(found_variant, context, artist_context, album_info) + return + logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}") + raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}") + + safe_move_file(file_path, final_path) + + if is_enhance_download and _enhance_source_info.get('original_file_path'): + original_enhance_path = _enhance_source_info['original_file_path'] + if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path): + try: + os.remove(original_enhance_path) + old_fmt = os.path.splitext(original_enhance_path)[1] + new_fmt = os.path.splitext(final_path)[1] + logger.info(f"[Enhance] Upgraded {old_fmt} → {new_fmt}: {os.path.basename(final_path)}") + except Exception as e: + logger.error(f"[Enhance] Could not remove old-format file: {e}") + elif is_enhance_download: + old_fmt = _enhance_source_info.get('original_format', 'unknown') + new_fmt = os.path.splitext(final_path)[1] + logger.info(f"[Enhance] Replaced in-place ({old_fmt} → {new_fmt}): {os.path.basename(final_path)}") + + download_cover_art(album_info, os.path.dirname(final_path), context) + generate_lrc_file(final_path, context, artist_context, album_info) + + if config_manager.get('post_processing.replaygain_enabled', False): + try: + from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF + if _rg_ffmpeg_ok(): + lufs, peak_dbfs = _rg_analyze(final_path) + gain_db = _RG_REF - lufs + _rg_write(final_path, gain_db, peak_dbfs) + pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}") + except Exception as rg_err: + pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") + + downsampled_path = downsample_hires_flac(final_path, context) + if downsampled_path: + final_path = downsampled_path + context['_final_processed_path'] = final_path + + blasphemy_path = create_lossy_copy(final_path) + if blasphemy_path: + context['_final_processed_path'] = blasphemy_path + + downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + cleanup_empty_directories(downloads_path, file_path) + + logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}") + + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + record_soulsync_library_entry(context, artist_context, album_info) + + try: + if not playlist_folder_mode: + completed_path = context.get('_final_processed_path', final_path) + record_retag_download(context, artist_context, album_info, completed_path) + except Exception as retag_err: + logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}") + + try: + completed_path = context.get('_final_processed_path', final_path) + batch_id_for_repair = context.get('batch_id') + if completed_path and batch_id_for_repair and repair_worker: + album_folder = os.path.dirname(str(completed_path)) + if album_folder: + repair_worker.register_folder(batch_id_for_repair, album_folder) + except Exception as repair_err: + logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}") + + try: + completed_path = context.get('_final_processed_path', final_path) + batch_id_for_consistency = context.get('batch_id') + if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'): + _file_info = { + 'path': str(completed_path), + 'track_number': album_info.get('track_number', 1), + 'disc_number': album_info.get('disc_number', 1), + 'title': get_import_clean_title( + context, + album_info=album_info, + default=album_info.get('clean_track_name', ''), + ), + } + with tasks_lock: + if batch_id_for_consistency in download_batches: + download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info) + except Exception as cons_err: + logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}") + + try: + check_and_remove_from_wishlist(context) + except Exception as wishlist_error: + logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}") + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id and batch_id: + logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['stream_processed'] = True + download_tasks[task_id]['status'] = 'completed' + logger.info(f"[Post-Process] Marked task {task_id} as completed") + _notify_download_completed(batch_id, task_id, success=True) + + except Exception as e: + import traceback + pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}") + pp_logger.info(traceback.format_exc()) + logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}") + traceback.print_exc() + + source_exists = os.path.exists(file_path) if file_path else False + if source_exists: + if context_key in _processed_download_ids: + _processed_download_ids.remove(context_key) + logger.warning(f"Removed {context_key} from processed set - will retry on next check") + with matched_context_lock: + if context_key not in matched_downloads_context: + matched_downloads_context[context_key] = context + logger.warning(f"Re-added {context_key} to context for retry") + else: + logger.warning(f"Source file gone, not retrying: {context_key}") + finally: + file_lock.release() + with _post_process_locks_lock: + _post_process_locks.pop(context_key, None) + + +def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime): + on_download_completed = getattr(runtime, "on_download_completed", None) + + def _notify_download_completed(batch_id, task_id, success=True): + if on_download_completed: + on_download_completed(batch_id, task_id, success=success) + + logger = pp_logger + try: + original_task_id = context.pop('task_id', None) + original_batch_id = context.pop('batch_id', None) + post_process_matched_download(context_key, context, file_path, runtime) + if original_task_id: + context['task_id'] = original_task_id + if original_batch_id: + context['batch_id'] = original_batch_id + + if context.get('_race_guard_failed'): + logger.info(f"Race guard: source file gone for task {task_id} — marking as failed") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task' + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + if context.get('_acoustid_quarantined'): + failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed') + logger.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}" + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + if context.get('_simple_download_completed'): + expected_final_path = context.get('_final_path') + if expected_final_path and os.path.exists(expected_final_path): + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=True) + return + logger.info( + f"FAILED simple download file not found at: {expected_final_path} " + f"(task={task_id}, context={context_key})" + ) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"Downloaded file not found at expected location: {os.path.basename(expected_final_path)}" + ) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + expected_final_path = context.get('_final_processed_path') + if not expected_final_path: + logger.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success") + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=True) + return + + if os.path.exists(expected_final_path): + redownload_ctx = None + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + download_tasks[task_id]['metadata_enhanced'] = True + redownload_ctx = download_tasks[task_id].get('_redownload_context') + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + if redownload_ctx: + try: + old_path = redownload_ctx.get('old_file_path') + lib_track_id = redownload_ctx.get('library_track_id') + if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path): + if os.path.normpath(old_path) != os.path.normpath(expected_final_path): + os.remove(old_path) + logger.info(f"[Redownload] Deleted old file: {old_path}") + if lib_track_id and expected_final_path: + _rd_db = get_database() + _rd_conn = _rd_db._get_connection() + _rd_cursor = _rd_conn.cursor() + _rd_cursor.execute( + """ + UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (expected_final_path, lib_track_id), + ) + _rd_conn.commit() + _rd_conn.close() + logger.info(f"[Redownload] Updated DB path for track {lib_track_id}") + except Exception as e: + logger.error(f"[Redownload] Post-processing hook error: {e}") + + _notify_download_completed(batch_id, task_id, success=True) + else: + track_name = get_import_clean_title(context, default=context_key) + logger.info(f"FAILED verification for '{track_name}' (task={task_id})") + logger.info(f" expected_final_path: {expected_final_path}") + logger.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}") + logger.info( + f" is_album={context.get('is_album_download', False)}, " + f"has_clean_data={get_import_has_clean_metadata(context)}" + ) + expected_dir = os.path.dirname(expected_final_path) + if os.path.exists(expected_dir): + dir_contents = os.listdir(expected_dir) + logger.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}") + else: + logger.info(f" directory does not exist: {expected_dir}") + + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing' + ) + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + _notify_download_completed(batch_id, task_id, success=False) + except Exception as e: + import traceback + logger.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}") + logger.info(traceback.format_exc()) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}" + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) diff --git a/core/import_runtime_state.py b/core/import_runtime_state.py new file mode 100644 index 00000000..9abf2baa --- /dev/null +++ b/core/import_runtime_state.py @@ -0,0 +1,121 @@ +"""Shared runtime state and tiny helpers for import/post-processing code.""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Dict, Optional + +from core.import_context import ( + build_import_album_info, + extract_artist_name, + get_import_clean_artist, + get_import_context_album, + get_import_original_search, + get_import_track_info, + normalize_import_context, +) + +matched_context_lock = threading.Lock() +matched_downloads_context: Dict[str, Dict[str, Any]] = {} +tasks_lock = threading.Lock() +download_tasks: Dict[str, Dict[str, Any]] = {} +download_batches: Dict[str, Dict[str, Any]] = {} +_processed_download_ids = set() +_post_process_locks: Dict[str, threading.Lock] = {} +_post_process_locks_lock = threading.Lock() + +activity_feed = [] +activity_feed_lock = threading.Lock() +_activity_toast_emitter = None + + +def set_activity_toast_emitter(emitter) -> None: + """Set the WebSocket-style emitter used by add_activity_item.""" + global _activity_toast_emitter + _activity_toast_emitter = emitter + + +def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True): + """Append an activity item and emit a toast if an emitter is configured.""" + activity_item = { + "icon": icon, + "title": title, + "subtitle": subtitle, + "time": time_ago, + "timestamp": time.time(), + "show_toast": show_toast, + } + with activity_feed_lock: + activity_feed.append(activity_item) + if len(activity_feed) > 20: + activity_feed.pop(0) + + if show_toast and _activity_toast_emitter is not None: + try: + _activity_toast_emitter("dashboard:toast", activity_item) + except Exception: + pass + + return activity_item + + +def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool: + """Mark a download task as completed in the shared task registry.""" + with tasks_lock: + task = download_tasks.get(task_id) + if not task: + return False + + task["status"] = "completed" + task["stream_processed"] = True + task["status_change_time"] = time.time() + if track_info is not None: + task["track_info"] = track_info + return True + + +def detect_album_info_web(context, artist_context=None): + """Best-effort album detection for single-track downloads.""" + context = normalize_import_context(context) + if artist_context is None: + artist_context = context.get("artist") or {} + + album_info = build_import_album_info(context) + if album_info.get("is_album"): + return album_info + + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + + album_name = ( + album_ctx.get("name") + or track_info.get("album") + or original_search.get("album") + or "" + ) + track_name = ( + track_info.get("name") + or original_search.get("title") + or "" + ) + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="") + + if album_name and track_name and album_name.strip().lower() not in { + track_name.strip().lower(), + artist_name.strip().lower(), + }: + return build_import_album_info( + context, + album_info={ + "album_name": album_name, + "track_number": track_info.get("track_number", 1), + "disc_number": track_info.get("disc_number", 1), + "album_image_url": album_ctx.get("image_url", ""), + "confidence": 0.5, + }, + force_album=True, + ) + + return None diff --git a/core/import_side_effects.py b/core/import_side_effects.py new file mode 100644 index 00000000..6c456e3d --- /dev/null +++ b/core/import_side_effects.py @@ -0,0 +1,567 @@ +"""Import post-processing side effects that do not need web runtime state.""" + +from __future__ import annotations + +import hashlib +import json +import os +from typing import Any, Dict, List, Optional + +from core.import_context import ( + extract_artist_name, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + normalize_import_context, + get_library_source_id_columns, +) +from core.wishlist_service import get_wishlist_service +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("import_side_effects") + + +def _get_config_manager(): + from config.settings import config_manager + + return config_manager + + +def _primary_track_artist_name(track_info: Dict[str, Any]) -> str: + artists = (track_info or {}).get("artists", []) + if isinstance(artists, list) and artists: + first = artists[0] + if isinstance(first, dict): + return str(first.get("name", "") or "") + return str(first or "") + if isinstance(artists, str): + return artists + return str((track_info or {}).get("artist", "") or "") + + +def _all_profile_wishlist_tracks(wishlist_service) -> List[Dict[str, Any]]: + database = get_database() + all_profiles = database.get_all_profiles() + wishlist_tracks: List[Dict[str, Any]] = [] + for profile in all_profiles: + wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=profile["id"])) + return wishlist_tracks + + +def _stable_soulsync_id(text: str) -> str: + return str(abs(int(hashlib.md5(text.encode("utf-8", errors="replace")).hexdigest(), 16)) % (10 ** 9)) + + +def emit_track_downloaded(context: Dict[str, Any], automation_engine=None) -> None: + """Emit the track_downloaded automation event.""" + try: + if not automation_engine: + return + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = "" + artists = ti.get("artists", []) + if artists: + first = artists[0] + artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first) + + automation_engine.emit( + "track_downloaded", + { + "artist": artist_name, + "title": ti.get("name", ti.get("title", "")), + "album": ti.get("album", ""), + "quality": context.get("_audio_quality", "Unknown"), + }, + ) + except Exception: + pass + + +def record_library_history_download(context: Dict[str, Any]) -> None: + """Record a completed download to the library_history table.""" + try: + search_result = context.get("original_search_result") or context.get("search_result") or {} + username = search_result.get("username", context.get("_download_username", "")) + source_map = { + "youtube": "YouTube", + "tidal": "Tidal", + "qobuz": "Qobuz", + "hifi": "HiFi", + "deezer_dl": "Deezer", + "lidarr": "Lidarr", + } + download_source = source_map.get(username, "Soulseek") + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = _primary_track_artist_name(ti) + if not artist_name: + artist_name = ti.get("artist", "") + + album_raw = ti.get("album", "") + album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "") + title = ti.get("name", ti.get("title", "")) + quality = context.get("_audio_quality", "") + file_path = context.get("_final_processed_path", context.get("_final_path", "")) + + thumb_url = "" + album_context = get_import_context_album(context) + if album_context: + thumb_url = album_context.get("image_url", "") + if not thumb_url: + images = album_context.get("images", []) + if images: + thumb_url = images[0].get("url", "") + if not thumb_url: + album_info = context.get("album_info", {}) + if isinstance(album_info, dict): + thumb_url = album_info.get("album_image_url", "") + + source_filename = search_result.get("filename", "") + source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "") + source_track_title = search_result.get("title", "") or search_result.get("name", "") + source_artist = search_result.get("artist", "") + if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr"): + stream_id = source_filename.split("||")[0] + if stream_id and not source_track_id: + source_track_id = stream_id + + acoustid_result = context.get("_acoustid_result", "") + + db = get_database() + db.add_library_history_entry( + event_type="download", + title=title, + artist_name=artist_name, + album_name=album_name, + quality=quality, + file_path=file_path, + thumb_url=thumb_url, + download_source=download_source, + source_track_id=source_track_id, + source_track_title=source_track_title, + source_filename=source_filename, + acoustid_result=acoustid_result, + source_artist=source_artist, + ) + except Exception: + pass + + +def record_download_provenance(context: Dict[str, Any]) -> None: + """Record source provenance for a completed download.""" + try: + search_result = context.get("original_search_result") or context.get("search_result") or {} + username = search_result.get("username", context.get("_download_username", "")) + filename = search_result.get("filename", "") + source_service = { + "youtube": "youtube", + "tidal": "tidal", + "qobuz": "qobuz", + "hifi": "hifi", + "deezer_dl": "deezer", + "lidarr": "lidarr", + }.get(username, "soulseek") + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = _primary_track_artist_name(ti) + if not artist_name: + artist_name = ti.get("artist", "") + + album_raw = ti.get("album", "") + album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "") + title = ti.get("name", ti.get("title", "")) + + file_path = context.get("_final_processed_path", context.get("_final_path", "")) + quality = context.get("_audio_quality", "") + size = search_result.get("size", 0) + + bit_depth = None + sample_rate = None + bitrate = None + try: + if file_path and os.path.isfile(file_path): + from mutagen import File as MutagenFile + + audio = MutagenFile(file_path) + if audio and audio.info: + sample_rate = getattr(audio.info, "sample_rate", None) + bitrate = getattr(audio.info, "bitrate", None) + bit_depth = getattr(audio.info, "bits_per_sample", None) + except Exception: + pass + + db = get_database() + db.record_track_download( + file_path=file_path, + source_service=source_service, + source_username=username, + source_filename=filename, + source_size=size or 0, + audio_quality=quality, + track_title=title, + track_artist=artist_name, + track_album=album_name, + bit_depth=bit_depth, + sample_rate=sample_rate, + bitrate=bitrate, + ) + except Exception: + pass + + +def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any]) -> None: + """Write imported media to the SoulSync library tables when the active server is SoulSync.""" + try: + config_manager = _get_config_manager() + if config_manager.get_active_media_server() != "soulsync": + return + + context = normalize_import_context(context) + final_path = context.get("_final_processed_path") + if not final_path: + return + + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + source_columns = get_library_source_id_columns(source) + + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="") + if not artist_name or artist_name in ("Unknown", "Unknown Artist"): + return + + album_name = "" + if album_info and isinstance(album_info, dict): + album_name = album_info.get("album_name", "") + if not album_name: + album_name = album_ctx.get("name", "") or original_search.get("album", "") + if not album_name: + album_name = track_info.get("name", "Unknown") + + track_name = get_import_clean_title( + context, + album_info=album_info, + default=track_info.get("name", "") or original_search.get("title", ""), + ) + track_number = (track_info.get("track_number") or (album_info.get("track_number") if isinstance(album_info, dict) else None)) or 1 + duration_ms = track_info.get("duration_ms", 0) or 0 + + year = None + release_date = album_ctx.get("release_date", "") + if release_date and len(release_date) >= 4: + try: + year = int(release_date[:4]) + except ValueError: + pass + + image_url = album_ctx.get("image_url", "") + if not image_url: + images = album_ctx.get("images", []) + if images and isinstance(images, list) and len(images) > 0: + img = images[0] + image_url = img.get("url", "") if isinstance(img, dict) else str(img) + + artist_source_id = source_ids.get("artist_id", "") + album_source_id = source_ids.get("album_id", "") + track_source_id = source_ids.get("track_id", "") + for key in ("auto_import", "from_sync_modal", "explicit_artist", "explicit_album", ""): + if artist_source_id == key: + artist_source_id = "" + if album_source_id == key: + album_source_id = "" + if track_source_id == key: + track_source_id = "" + + genres = (artist_context or {}).get("genres", []) if isinstance(artist_context, dict) else [] + if genres: + from core.genre_filter import filter_genres as _filter_genres + + genres = _filter_genres(genres, config_manager) + genres_json = json.dumps(genres) if genres else "" + + bitrate = 0 + try: + from mutagen import File as MutagenFile + + audio = MutagenFile(final_path) + if audio and hasattr(audio, "info") and audio.info and hasattr(audio.info, "bitrate"): + bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0 + except Exception: + pass + + artist_id = _stable_soulsync_id(artist_name.lower().strip()) + album_id = _stable_soulsync_id(f"{artist_name}::{album_name}".lower().strip()) + track_id = _stable_soulsync_id(final_path) + total_tracks = album_ctx.get("total_tracks", 0) or 0 + + db = get_database() + with db._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,)) + if not cursor.fetchone(): + cursor.execute( + "SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1", + (artist_name,), + ) + existing_by_name = cursor.fetchone() + if existing_by_name: + artist_id = existing_by_name[0] + else: + cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,)) + if cursor.fetchone(): + artist_id = _stable_soulsync_id(artist_name.lower().strip() + "::soulsync") + cursor.execute( + """ + INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at) + VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (artist_id, artist_name, genres_json, image_url), + ) + artist_source_col = source_columns.get("artist") + if artist_source_col and artist_source_id: + try: + cursor.execute( + f"UPDATE artists SET {artist_source_col} = ? WHERE id = ?", + (artist_source_id, artist_id), + ) + except Exception: + pass + + cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,)) + if not cursor.fetchone(): + cursor.execute( + "SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1", + (album_name, artist_id), + ) + existing_album_by_name = cursor.fetchone() + if existing_album_by_name: + album_id = existing_album_by_name[0] + else: + cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,)) + if cursor.fetchone(): + album_id = _stable_soulsync_id(f"{artist_name}::{album_name}::soulsync".lower().strip()) + cursor.execute( + """ + INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count, + duration, server_source, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms), + ) + album_source_col = source_columns.get("album") + if album_source_col and album_source_id: + try: + cursor.execute( + f"UPDATE albums SET {album_source_col} = ? WHERE id = ?", + (album_source_id, album_id), + ) + except Exception: + pass + + track_artist = None + track_artists_list = track_info.get("artists", []) or original_search.get("artists", []) + if track_artists_list: + first_track_artist = track_artists_list[0] + if isinstance(first_track_artist, dict): + ta_name = first_track_artist.get("name", "") + else: + ta_name = str(first_track_artist) + if ta_name and ta_name.lower() != artist_name.lower(): + track_artist = ta_name + + cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,)) + if not cursor.fetchone(): + cursor.execute( + """ + INSERT INTO tracks (id, album_id, artist_id, title, track_number, + duration, file_path, bitrate, track_artist, server_source, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + ( + track_id, + album_id, + artist_id, + track_name, + track_number, + duration_ms, + final_path, + bitrate, + track_artist, + ), + ) + track_source_col = source_columns.get("track") + if track_source_col and track_source_id: + try: + cursor.execute( + f"UPDATE tracks SET {track_source_col} = ? WHERE id = ?", + (track_source_id, track_id), + ) + track_album_col = source_columns.get("track_album") + if track_album_col and album_source_id: + cursor.execute( + f"UPDATE tracks SET {track_album_col} = ? WHERE id = ?", + (album_source_id, track_id), + ) + except Exception: + pass + + conn.commit() + logger.info("[SoulSync Library] Added: %s / %s / %s", artist_name, album_name, track_name) + except Exception as exc: + logger.error("[SoulSync Library] Could not record library entry: %s", exc) + + +def record_retag_download(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any], final_path: str) -> None: + """Record a completed download for later re-tagging.""" + try: + db = get_database() + + context = normalize_import_context(context) + artist_context = get_import_context_artist(context) or (artist_context if isinstance(artist_context, dict) else {}) + album_context = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="Unknown Artist") + is_album = album_info and album_info.get("is_album", False) + group_type = "album" if is_album else "single" + album_name = album_info.get("album_name", "") if album_info else get_import_clean_album(context, default=original_search.get("album", "Unknown")) + + image_url = album_info.get("album_image_url") if album_info else None + if not image_url: + image_url = album_context.get("image_url", "") + if not image_url and album_context.get("images"): + images = album_context.get("images", []) + if images and isinstance(images[0], dict): + image_url = images[0].get("url", "") + + total_tracks = album_context.get("total_tracks", 1) if album_context else 1 + release_date = album_context.get("release_date", "") if album_context else "" + + spotify_album_id = None + itunes_album_id = None + if source == "spotify": + spotify_album_id = source_ids.get("album_id", "") or None + elif source == "itunes": + itunes_album_id = source_ids.get("album_id", "") or None + + group_id = db.find_retag_group(artist_name, album_name) + if group_id is None: + group_id = db.add_retag_group( + group_type=group_type, + artist_name=artist_name, + album_name=album_name, + image_url=image_url, + spotify_album_id=spotify_album_id, + itunes_album_id=itunes_album_id, + total_tracks=total_tracks, + release_date=release_date, + ) + if group_id is None: + return + + track_number = album_info.get("track_number", 1) if album_info else (track_info.get("track_number", 1) or 1) + disc_number = original_search.get("disc_number") or (album_info.get("disc_number", 1) if album_info else track_info.get("disc_number", 1) or 1) + title = get_import_clean_title( + context, + album_info=album_info, + default=album_info.get("clean_track_name", "Unknown Track") if album_info else "Unknown Track", + ) + file_format = os.path.splitext(str(final_path))[1].lstrip(".").lower() + + spotify_track_id = None + itunes_track_id = None + if source == "spotify": + spotify_track_id = source_ids.get("track_id", "") or None + elif source == "itunes": + itunes_track_id = source_ids.get("track_id", "") or None + + if not db.retag_track_exists(group_id, str(final_path)): + db.add_retag_track( + group_id=group_id, + track_number=track_number, + disc_number=disc_number, + title=title, + file_path=str(final_path), + file_format=file_format, + spotify_track_id=spotify_track_id, + itunes_track_id=itunes_track_id, + ) + logger.info("[Retag] Recorded track for retag: '%s' in '%s'", title, album_name) + + db.trim_retag_groups(100) + except Exception as exc: + logger.error("[Retag] Could not record track for retag: %s", exc) + + +def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None: + """Check whether a successful download should be removed from the wishlist.""" + try: + wishlist_service = get_wishlist_service() + spotify_track_id = None + + track_info = context.get("track_info", {}) + if track_info.get("id"): + spotify_track_id = track_info["id"] + logger.info("[Wishlist] Found Spotify ID from track_info: %s", spotify_track_id) + elif context.get("original_search_result", {}).get("id"): + spotify_track_id = context["original_search_result"]["id"] + logger.info("[Wishlist] Found Spotify ID from original_search_result: %s", spotify_track_id) + elif "wishlist_id" in track_info: + wishlist_id = track_info["wishlist_id"] + logger.info("[Wishlist] Found wishlist_id in context: %s", wishlist_id) + wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service) + for wishlist_track in wishlist_tracks: + if wishlist_track.get("wishlist_id") == wishlist_id: + spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") + logger.info("[Wishlist] Found Spotify ID from wishlist entry: %s", spotify_track_id) + break + + if not spotify_track_id: + track_name = track_info.get("name") or context.get("original_search_result", {}).get("title", "") + artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(context.get("original_search_result", {})) + + if track_name and artist_name: + logger.warning("[Wishlist] No Spotify ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name) + + wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service) + for wishlist_track in wishlist_tracks: + wl_name = wishlist_track.get("name", "").lower() + wl_artists = wishlist_track.get("artists", []) + wl_artist_name = "" + if wl_artists: + if isinstance(wl_artists[0], dict): + wl_artist_name = wl_artists[0].get("name", "").lower() + else: + wl_artist_name = str(wl_artists[0]).lower() + if wl_name == track_name.lower() and wl_artist_name == artist_name.lower(): + spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") + logger.info("[Wishlist] Found fuzzy match - Spotify ID: %s", spotify_track_id) + break + + if spotify_track_id: + logger.info("[Wishlist] Attempting to remove track from wishlist: %s", spotify_track_id) + removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) + if removed: + logger.info("[Wishlist] Successfully removed track from wishlist: %s", spotify_track_id) + else: + logger.warning("ℹ️ [Wishlist] Track not found in wishlist or already removed: %s", spotify_track_id) + else: + logger.warning("ℹ️ [Wishlist] No Spotify track ID found for wishlist removal check") + except Exception as exc: + logger.error("[Wishlist] Error in wishlist removal check: %s", exc) diff --git a/core/import_staging.py b/core/import_staging.py new file mode 100644 index 00000000..da93580a --- /dev/null +++ b/core/import_staging.py @@ -0,0 +1,454 @@ +"""Shared staging folder and import suggestion helpers.""" + +from __future__ import annotations + +import os +import threading +from typing import Any, Dict, List, Optional, Tuple + +from core.import_paths import docker_resolve_path +from utils.logging_config import get_logger + +logger = get_logger("import_staging") + +AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"} + +_import_suggestions_cache_lock = threading.Lock() +_import_suggestions_cache: Dict[str, Any] = { + "suggestions": [], + "building": False, + "built": False, +} + + +def _get_config_manager(): + try: + from config.settings import config_manager + + return config_manager + except Exception: + class _FallbackConfig: + @staticmethod + def get(key, default=None): + return default + + return _FallbackConfig() + + +def get_staging_path() -> str: + """Resolve the configured staging folder path.""" + raw = _get_config_manager().get("import.staging_path", "./Staging") + return docker_resolve_path(raw) + + +def get_import_suggestions_cache() -> Dict[str, Any]: + """Expose the shared import suggestions cache.""" + return _import_suggestions_cache + + +def get_primary_source() -> str: + from core.metadata_service import get_primary_source as _get_primary_source + + return _get_primary_source() + + +def get_source_priority(preferred_source: str): + from core.metadata_service import get_source_priority as _get_source_priority + + return _get_source_priority(preferred_source) + + +def get_client_for_source(source: str): + from core.metadata_service import get_client_for_source as _get_client_for_source + + return _get_client_for_source(source) + + +def _search_albums_for_source(source: str, client: Any, query: str, limit: int = 5): + from core.metadata_service import _search_albums_for_source as _metadata_search_albums_for_source + + return _metadata_search_albums_for_source(source, client, query, limit=limit) + + +def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 5): + from core.metadata_service import _search_tracks_for_source as _metadata_search_tracks_for_source + + return _metadata_search_tracks_for_source(source, client, query, limit=limit) + + +def _extract_value(value: Any, *names: str, default: Any = None) -> Any: + if value is None: + return default + + if isinstance(value, (str, bytes)): + return default + + for name in names: + if isinstance(value, dict): + if name in value and value[name] is not None: + return value[name] + else: + candidate = getattr(value, name, None) + if candidate is not None: + return candidate + + return default + + +def _extract_artist_names(artists: Any) -> List[str]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artist = str(artists).strip() + return [artist] if artist else [] + + try: + items = list(artists) + except TypeError: + items = [artists] + + names: List[str] = [] + for artist in items: + if isinstance(artist, dict): + name = str(_extract_value(artist, "name", "artist_name", "title", default="") or "").strip() + else: + candidate = getattr(artist, "name", None) + if candidate is None: + candidate = artist + name = str(candidate or "").strip() + if name: + names.append(name) + + return names + + +def _normalize_album_result(album: Any, source: str) -> Dict[str, Any]: + album_id = str(_extract_value(album, "id", "album_id", "release_id", default="") or "").strip() + album_name = str(_extract_value(album, "name", "title", default="") or "").strip() + artists = _extract_artist_names(_extract_value(album, "artists", default=[])) + artist_name = ", ".join(artists) if artists else str( + _extract_value(album, "artist_name", "artist", default="Unknown Artist") or "Unknown Artist" + ).strip() + release_date = str(_extract_value(album, "release_date", "releaseDate", default="") or "").strip() + album_type = str(_extract_value(album, "album_type", "type", default="album") or "album").strip() or "album" + + total_tracks = _extract_value(album, "total_tracks", "track_count", default=0) + if isinstance(total_tracks, (list, tuple, set)): + total_tracks = len(total_tracks) + try: + total_tracks = int(total_tracks or 0) + except (TypeError, ValueError): + total_tracks = 0 + + image_url = _extract_value(album, "image_url", "thumb_url", "cover_image", "cover_url", default="") + if not image_url: + images = _extract_value(album, "images", default=[]) or [] + if isinstance(images, dict): + images = [images] + elif isinstance(images, (str, bytes)): + images = [images] + try: + images = list(images) + except TypeError: + images = [images] + if images: + first_image = images[0] + if isinstance(first_image, (str, bytes)): + image_url = str(first_image).strip() + else: + image_url = _extract_value(first_image, "url", "image_url", "src", default="") + + return { + "id": album_id or album_name or "unknown-album", + "name": album_name or album_id or "Unknown Album", + "artist": artist_name or "Unknown Artist", + "release_date": release_date, + "total_tracks": total_tracks, + "image_url": str(image_url or ""), + "album_type": album_type, + "source": source, + } + + +def _album_fingerprint(album: Dict[str, Any]) -> Tuple[str, str, str, str]: + return ( + str(album.get("name", "") or "").strip().casefold(), + str(album.get("artist", "") or "").strip().casefold(), + str(album.get("release_date", "") or "").strip()[:10].casefold(), + str(album.get("album_type", "") or "").strip().casefold(), + ) + + +def _normalize_track_result(track: Any, source: str) -> Dict[str, Any]: + track_id = str(_extract_value(track, "id", "track_id", "trackId", default="") or "").strip() + track_name = str(_extract_value(track, "name", "title", "track_name", default="") or "").strip() + artists = _extract_artist_names(_extract_value(track, "artists", default=[])) + artist_name = ", ".join(artists) if artists else str( + _extract_value(track, "artist", "artist_name", default="Unknown Artist") or "Unknown Artist" + ).strip() + + album_value = _extract_value(track, "album", default=None) + album_name = "" + album_id = str(_extract_value(track, "album_id", "collectionId", "albumId", default="") or "").strip() + if isinstance(album_value, dict): + album_name = str(_extract_value(album_value, "name", "title", default="") or "").strip() + album_id = album_id or str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip() + if not album_name: + album_name = album_id + elif isinstance(album_value, (str, bytes)): + album_name = str(album_value).strip() + elif album_value is not None: + album_name = str(_extract_value(album_value, "name", "title", default=album_value) or "").strip() + if not album_id: + album_id = str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip() + + image_url = _extract_value(track, "image_url", "thumb_url", "cover_image", default="") + if not image_url: + images = _extract_value(track, "images", default=[]) or [] + if isinstance(images, dict): + images = [images] + elif isinstance(images, (str, bytes)): + images = [images] + try: + images = list(images) + except TypeError: + images = [images] + if images: + first_image = images[0] + if isinstance(first_image, (str, bytes)): + image_url = str(first_image).strip() + else: + image_url = _extract_value(first_image, "url", "image_url", "src", default="") + if not image_url and album_value is not None: + album_images = _extract_value(album_value, "images", default=[]) or [] + if isinstance(album_images, dict): + album_images = [album_images] + elif isinstance(album_images, (str, bytes)): + album_images = [album_images] + try: + album_images = list(album_images) + except TypeError: + album_images = [album_images] + if album_images: + first_album_image = album_images[0] + if isinstance(first_album_image, (str, bytes)): + image_url = str(first_album_image).strip() + else: + image_url = _extract_value(first_album_image, "url", "image_url", "src", default="") + + duration_ms = _extract_value(track, "duration_ms", "duration", "trackTimeMillis", default=0) + try: + duration_ms = int(duration_ms or 0) + except (TypeError, ValueError): + duration_ms = 0 + + track_number = _extract_value(track, "track_number", "trackNumber", default=1) + try: + track_number = int(track_number or 1) + except (TypeError, ValueError): + track_number = 1 + + return { + "id": track_id or track_name or "unknown-track", + "name": track_name or track_id or "Unknown Track", + "artist": artist_name or "Unknown Artist", + "album": album_name or "", + "album_id": album_id or "", + "duration_ms": duration_ms, + "image_url": str(image_url or ""), + "track_number": track_number, + "source": source, + } + + +def _read_staging_audio_tags(file_path: str) -> Tuple[Optional[str], Optional[str]]: + try: + from mutagen import File as MutagenFile + + tags = MutagenFile(file_path, easy=True) + if not tags: + return None, None + + album = (tags.get("album") or [None])[0] + artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0] + album_text = str(album).strip() if album else "" + artist_text = str(artist).strip() if artist else "" + return (album_text or None, artist_text or None) + except Exception: + return None, None + + +def _collect_import_suggestion_queries(staging_path: str) -> List[str]: + tag_albums: Dict[Tuple[str, str], int] = {} + folder_hints: Dict[str, int] = {} + + for root, _dirs, filenames in os.walk(staging_path): + audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] + if not audio_files: + continue + + rel_dir = os.path.relpath(root, staging_path) + if rel_dir != ".": + top_folder = rel_dir.split(os.sep)[0] + folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) + + for fname in audio_files: + full_path = os.path.join(root, fname) + album, artist = _read_staging_audio_tags(full_path) + if album: + key = (album.strip(), (artist or "").strip()) + tag_albums[key] = tag_albums.get(key, 0) + 1 + + queries: List[str] = [] + seen_lower = set() + + for (album, artist), _count in sorted(tag_albums.items(), key=lambda item: -item[1]): + q = f"{album} {artist}".strip() if artist else album + if q and q.lower() not in seen_lower: + seen_lower.add(q.lower()) + queries.append(q) + + for folder, _count in sorted(folder_hints.items(), key=lambda item: -item[1]): + q = folder.replace("_", " ") + if q and q.lower() not in seen_lower: + seen_lower.add(q.lower()) + queries.append(q) + + return queries[:5] + + +def search_import_albums(query: str, limit: int = 12) -> List[Dict[str, Any]]: + """Search albums using the configured metadata provider first.""" + query = (query or "").strip() + if not query: + return [] + + results: List[Dict[str, Any]] = [] + seen = set() + source_chain = get_source_priority(get_primary_source()) + + for source in source_chain: + client = get_client_for_source(source) + if not client: + continue + + source_results = _search_albums_for_source(source, client, query, limit=limit) + if not source_results: + continue + + added_for_source = False + for album in source_results: + suggestion = _normalize_album_result(album, source) + fingerprint = _album_fingerprint(suggestion) + if fingerprint in seen: + continue + seen.add(fingerprint) + results.append(suggestion) + added_for_source = True + if len(results) >= limit: + return results[:limit] + + if added_for_source: + break + + return results[:limit] + + +def search_import_tracks(query: str, limit: int = 30) -> List[Dict[str, Any]]: + """Search tracks using the configured metadata provider priority order.""" + query = (query or "").strip() + if not query: + return [] + + results: List[Dict[str, Any]] = [] + source_chain = get_source_priority(get_primary_source()) + + for source in source_chain: + client = get_client_for_source(source) + if not client: + continue + + source_results = _search_tracks_for_source(source, client, query, limit=limit) + if not source_results: + continue + + for track in source_results: + results.append(_normalize_track_result(track, source)) + if len(results) >= limit: + return results[:limit] + break + + return results[:limit] + + +def _build_import_suggestions_background(): + cache = _import_suggestions_cache + + with _import_suggestions_cache_lock: + if cache["building"]: + return + cache["building"] = True + + try: + staging_path = get_staging_path() + if not os.path.isdir(staging_path): + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + return + + queries = _collect_import_suggestion_queries(staging_path) + if not queries: + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + return + + suggestions: List[Dict[str, Any]] = [] + seen = set() + for query in queries: + try: + albums = search_import_albums(query, limit=2) + for album in albums: + fingerprint = _album_fingerprint(album) + if fingerprint in seen: + continue + seen.add(fingerprint) + suggestions.append(album) + except Exception as exc: + logger.warning("Import suggestion search failed for %r: %s", query, exc) + + with _import_suggestions_cache_lock: + cache["suggestions"] = suggestions[:8] + cache["built"] = True + + logger.info( + "Import suggestions cache built: %s suggestions from %s hints", + len(cache["suggestions"]), + len(queries), + ) + except Exception as exc: + logger.error("Error building import suggestions cache: %s", exc) + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + finally: + with _import_suggestions_cache_lock: + cache["building"] = False + + +def start_import_suggestions_cache(): + """Start building the import suggestions cache in a background thread.""" + threading.Thread( + target=_build_import_suggestions_background, + daemon=True, + name="import-suggestions-cache", + ).start() + + +def refresh_import_suggestions_cache(): + """Invalidate and rebuild the suggestions cache.""" + with _import_suggestions_cache_lock: + _import_suggestions_cache["built"] = False + start_import_suggestions_cache() diff --git a/core/metadata_enrichment.py b/core/metadata_enrichment.py new file mode 100644 index 00000000..293af162 --- /dev/null +++ b/core/metadata_enrichment.py @@ -0,0 +1,1486 @@ +"""Source-aware metadata enrichment helpers for imported audio files.""" + +from __future__ import annotations + +import json +import os +import re +import threading +import time +import urllib.request +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, Optional + +from core.import_context import ( + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + get_source_tag_names, + normalize_import_context, +) +from config.settings import config_manager +from core.metadata_service import get_itunes_client +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("metadata_enrichment") + +_FILE_LOCKS: Dict[str, threading.Lock] = {} +_FILE_LOCKS_LOCK = threading.Lock() + +_MB_RELEASE_CACHE: Dict[tuple, str] = {} +_MB_RELEASE_CACHE_LOCK = threading.RLock() +_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} +_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() + +_EDITION_PAREN_RE = re.compile( + r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]', + re.IGNORECASE, +) +_EDITION_BARE_RE = re.compile( + r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?\s*$', + re.IGNORECASE, +) + + +class _NullConfigManager: + def get(self, _key: str, default: Any = None) -> Any: + return default + + +def _get_logger(runtime=None): + return logger + + +def _get_config_manager(runtime=None): + return config_manager + + +def _get_database(runtime=None): + try: + return get_database() + except Exception: + return None + + +def _get_itunes_client(runtime=None): + try: + return get_itunes_client() + except Exception: + worker = getattr(runtime, "itunes_enrichment_worker", None) + if worker and getattr(worker, "client", None): + return worker.client + return getattr(runtime, "itunes_client", None) + + +def _extract_artist_name(artist: Any) -> str: + if isinstance(artist, dict): + return str(artist.get("name", "") or "") + if hasattr(artist, "name"): + return str(getattr(artist, "name") or "") + return str(artist) if artist else "" + + +def _get_mutagen_symbols(runtime=None): + """Lazy mutagen import so tests can monkeypatch this without the package installed.""" + try: + from mutagen import File as MutagenFile + from mutagen.apev2 import APEv2, APENoHeaderError + from mutagen.flac import FLAC, Picture + from mutagen.id3 import ( + APIC, + ID3, + TBPM, + TCOP, + TDOR, + TDRC, + TCON, + TIT2, + TALB, + TPE1, + TPE2, + TPOS, + TPUB, + TRCK, + TSRC, + TXXX, + UFID, + TMED, + ) + from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm + from mutagen.oggvorbis import OggVorbis + try: + from mutagen.oggopus import OggOpus + except Exception: + OggOpus = None + except Exception as exc: + _get_logger(runtime).debug("Mutagen unavailable for metadata enrichment: %s", exc) + return None + + return SimpleNamespace( + File=MutagenFile, + APEv2=APEv2, + APENoHeaderError=APENoHeaderError, + FLAC=FLAC, + Picture=Picture, + ID3=ID3, + APIC=APIC, + TBPM=TBPM, + TCOP=TCOP, + TDOR=TDOR, + TDRC=TDRC, + TCON=TCON, + TIT2=TIT2, + TALB=TALB, + TPE1=TPE1, + TPE2=TPE2, + TPOS=TPOS, + TPUB=TPUB, + TRCK=TRCK, + TSRC=TSRC, + TXXX=TXXX, + UFID=UFID, + TMED=TMED, + MP4=MP4, + MP4Cover=MP4Cover, + MP4FreeForm=MP4FreeForm, + OggVorbis=OggVorbis, + OggOpus=OggOpus, + ) + + +def _get_file_lock(file_path: str) -> threading.Lock: + with _FILE_LOCKS_LOCK: + lock = _FILE_LOCKS.get(file_path) + if lock is None: + lock = threading.Lock() + _FILE_LOCKS[file_path] = lock + return lock + + +def _is_ogg_opus(audio_file: Any) -> bool: + return type(audio_file).__name__ == "OggOpus" + + +def _is_vorbis_like(audio_file: Any, symbols: Any) -> bool: + vorbis_classes = tuple( + cls for cls in ( + getattr(symbols, "FLAC", None), + getattr(symbols, "OggVorbis", None), + ) if cls is not None + ) + return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or _is_ogg_opus(audio_file) + + +def _save_audio_file(audio_file: Any, symbols: Any) -> None: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.save(v1=0, v2_version=4) + elif isinstance(audio_file, symbols.FLAC): + audio_file.save(deleteid3=True) + else: + audio_file.save() + + +def _strip_all_non_audio_tags(file_path: str, runtime=None) -> dict: + summary = {"apev2_stripped": False, "apev2_tag_count": 0} + if os.path.splitext(file_path)[1].lower() != ".mp3": + return summary + + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return summary + + try: + apev2_tags = symbols.APEv2(file_path) + tag_count = len(apev2_tags) + tag_keys = list(apev2_tags.keys()) + apev2_tags.delete(file_path) + summary["apev2_stripped"] = True + summary["apev2_tag_count"] = tag_count + _get_logger(runtime).info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10])) + except symbols.APENoHeaderError: + pass + except Exception as exc: + _get_logger(runtime).error("Could not strip APEv2 tags (non-fatal): %s", exc) + return summary + + +def _verify_metadata_written(file_path: str, runtime=None) -> bool: + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return False + + try: + check = symbols.File(file_path) + if check is None or check.tags is None: + _get_logger(runtime).info("[VERIFY] Tags are None after save: %s", file_path) + return False + + title_found = False + artist_found = False + if isinstance(check.tags, symbols.ID3): + title_found = bool(check.tags.getall("TIT2")) + artist_found = bool(check.tags.getall("TPE1")) + try: + symbols.APEv2(file_path) + _get_logger(runtime).info("[VERIFY] APEv2 tags still present after processing!") + return False + except symbols.APENoHeaderError: + pass + elif _is_vorbis_like(check, symbols): + title_found = bool(check.get("title")) + artist_found = bool(check.get("artist")) + elif isinstance(check, symbols.MP4): + title_found = bool(check.get("\xa9nam")) + artist_found = bool(check.get("\xa9ART")) + + if not title_found or not artist_found: + _get_logger(runtime).warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found) + return False + + _get_logger(runtime).info("[VERIFY] Metadata verified OK") + return True + except Exception as exc: + _get_logger(runtime).error("[VERIFY] Verification error (non-fatal): %s", exc) + return False + + +def _get_image_dimensions(data: bytes): + try: + if data[:8] == b"\x89PNG\r\n\x1a\n": + import struct + + w, h = struct.unpack(">II", data[16:24]) + return w, h + if data[:2] == b"\xff\xd8": + import struct + + i = 2 + while i < len(data) - 9: + if data[i] != 0xFF: + break + marker = data[i + 1] + if marker in (0xC0, 0xC2): + h, w = struct.unpack(">HH", data[i + 5 : i + 9]) + return w, h + length = struct.unpack(">H", data[i + 2 : i + 4])[0] + i += 2 + length + except Exception: + pass + return None, None + + +def extract_source_metadata(context: dict, artist: dict, album_info: dict, runtime=None) -> dict: + if album_info is None: + album_info = {} + + cfg = _get_config_manager(runtime) + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + + artist_dict = artist if isinstance(artist, dict) else { + "name": _extract_artist_name(artist), + "id": getattr(artist, "id", ""), + "genres": list(getattr(artist, "genres", []) or []), + } + + metadata: Dict[str, Any] = { + "source": source, + "source_track_id": source_ids["track_id"], + "source_artist_id": source_ids["artist_id"], + "source_album_id": source_ids["album_id"], + } + + metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "")) + if original_search.get("clean_title"): + _get_logger(runtime).info("Metadata: Using clean title: '%s'", metadata["title"]) + elif album_info.get("clean_track_name"): + _get_logger(runtime).info("Metadata: Using album info clean name: '%s'", metadata["title"]) + else: + _get_logger(runtime).warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) + + artists = original_search.get("artists") + if isinstance(artists, list) and artists: + all_artists = [] + for artist_item in artists: + if isinstance(artist_item, dict) and artist_item.get("name"): + all_artists.append(artist_item["name"]) + elif isinstance(artist_item, str): + all_artists.append(artist_item) + else: + all_artists.append(str(artist_item)) + metadata["artist"] = ", ".join(all_artists) + _get_logger(runtime).info("Metadata: Using all artists: '%s'", metadata["artist"]) + else: + metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context) + _get_logger(runtime).info("Metadata: Using primary artist: '%s'", metadata["artist"]) + + raw_album_artist = artist_dict.get("name", "") or metadata["artist"] + track_info_ctx = track_info or {} + explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None + album_artists_for_collab = None + + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + raw_album_artist = explicit_artist["name"] + album_artists_for_collab = [explicit_artist] + elif isinstance(explicit_artist, str) and explicit_artist: + raw_album_artist = explicit_artist + album_artists_for_collab = [{"name": explicit_artist}] + elif album_ctx and isinstance(album_ctx, dict): + album_artists = album_ctx.get("artists", []) + if album_artists: + first_album_artist = album_artists[0] + if isinstance(first_album_artist, dict) and first_album_artist.get("name"): + raw_album_artist = first_album_artist["name"] + elif isinstance(first_album_artist, str) and first_album_artist: + raw_album_artist = first_album_artist + album_artists_for_collab = album_artists + + collab_mode = cfg.get("file_organization.collab_artist_mode", "first") + if collab_mode == "first" and raw_album_artist: + context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or [] + if len(context_artists) > 1: + first = context_artists[0] + raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first) + elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist): + artist_id = str(artist_dict.get("id", "")) + if source == "itunes" and artist_id.isdigit(): + try: + itunes_client = _get_itunes_client(runtime) + if itunes_client and hasattr(itunes_client, "resolve_primary_artist"): + resolved = itunes_client.resolve_primary_artist(artist_id) + if resolved and resolved != raw_album_artist: + raw_album_artist = resolved + except Exception: + pass + metadata["album_artist"] = raw_album_artist + + if album_info.get("is_album"): + metadata["album"] = album_info.get("album_name", "Unknown Album") + metadata["track_number"] = album_info.get("track_number", 1) + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1 + _get_logger(runtime).info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) + else: + if album_ctx and album_ctx.get("name"): + _get_logger(runtime).info("[SAFEGUARD] Using album context name instead of track title for album metadata") + metadata["album"] = album_ctx["name"] + metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1 + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) + else: + metadata["album"] = metadata["title"] + metadata["track_number"] = 1 + metadata["total_tracks"] = 1 + + disc_num = original_search.get("disc_number") + if disc_num is None and album_info: + disc_num = album_info.get("disc_number") + metadata["disc_number"] = disc_num if disc_num is not None else 1 + + if album_ctx and album_ctx.get("release_date"): + metadata["date"] = album_ctx["release_date"][:4] + + genres = artist_dict.get("genres") or [] + if genres: + from core.genre_filter import filter_genres + + filtered = filter_genres(list(genres[:2]), cfg) + if filtered: + metadata["genre"] = ", ".join(filtered) + + metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None + if not metadata["album_art_url"] and album_ctx: + album_image = album_ctx.get("image_url") + if not album_image and album_ctx.get("images"): + first_image = album_ctx["images"][0] + album_image = first_image.get("url") if isinstance(first_image, dict) else None + metadata["album_art_url"] = album_image + + _get_logger(runtime).info( + "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", + metadata.get("title"), + metadata.get("artist"), + metadata.get("album_artist"), + metadata.get("album"), + metadata.get("track_number"), + metadata.get("total_tracks"), + metadata.get("disc_number"), + ) + + return metadata + + +def embed_album_art_metadata(audio_file, metadata: dict, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return + + try: + image_data = None + mime_type = None + + release_mbid = metadata.get("musicbrainz_release_id") + if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False): + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if not image_data: + art_url = metadata.get("album_art_url") + if not art_url: + logger_.warning("No album art URL available for embedding.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + + if not image_data: + logger_.error("Failed to download album art data.") + return + + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data)) + elif isinstance(audio_file, symbols.FLAC): + picture = symbols.Picture() + picture.data = image_data + picture.type = 3 + picture.mime = mime_type + width, height = _get_image_dimensions(image_data) + picture.width = width or 640 + picture.height = height or 640 + picture.depth = 24 + audio_file.add_picture(picture) + elif isinstance(audio_file, symbols.MP4): + fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG + audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] + + logger_.info("Album art successfully embedded.") + except Exception as exc: + logger_.error("Error embedding album art: %s", exc) + + +def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return + + try: + tag_config = { + "SPOTIFY_TRACK_ID": "spotify.tags.track_id", + "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", + "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", + "ITUNES_TRACK_ID": "itunes.tags.track_id", + "ITUNES_ARTIST_ID": "itunes.tags.artist_id", + "ITUNES_ALBUM_ID": "itunes.tags.album_id", + "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", + "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", + "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", + "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", + "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", + "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", + "RELEASETYPE": "musicbrainz.tags.release_type", + "ORIGINALDATE": "musicbrainz.tags.original_date", + "RELEASESTATUS": "musicbrainz.tags.release_status", + "RELEASECOUNTRY": "musicbrainz.tags.release_country", + "BARCODE": "musicbrainz.tags.barcode", + "MEDIA": "musicbrainz.tags.media", + "TOTALDISCS": "musicbrainz.tags.total_discs", + "CATALOGNUMBER": "musicbrainz.tags.catalog_number", + "SCRIPT": "musicbrainz.tags.script", + "ASIN": "musicbrainz.tags.asin", + "DEEZER_TRACK_ID": "deezer.tags.track_id", + "DEEZER_ARTIST_ID": "deezer.tags.artist_id", + "AUDIODB_TRACK_ID": "audiodb.tags.track_id", + "TIDAL_TRACK_ID": "tidal.tags.track_id", + "TIDAL_ARTIST_ID": "tidal.tags.artist_id", + "QOBUZ_TRACK_ID": "qobuz.tags.track_id", + "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", + "GENIUS_TRACK_ID": "genius.tags.track_id", + } + + def _tag_enabled(path: str) -> bool: + return cfg.get(path, True) is not False + + def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: + if not a or not b: + return False + from difflib import SequenceMatcher + + norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() + return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold + + context = normalize_import_context(context) + source = (metadata.get("source") or "").strip().lower() + source_ids = {} + if source: + source_tag_names = get_source_tag_names(source) + source_track_id = metadata.get("source_track_id") + source_artist_id = metadata.get("source_artist_id") + source_album_id = metadata.get("source_album_id") + if cfg.get(f"{source}.embed_tags", True) is not False: + if source_tag_names.get("track") and source_track_id: + source_ids[source_tag_names["track"]] = source_track_id + if source_tag_names.get("artist") and source_artist_id: + source_ids[source_tag_names["artist"]] = source_artist_id + if source_tag_names.get("album") and source_album_id: + source_ids[source_tag_names["album"]] = source_album_id + + if not source_ids: + if cfg.get("spotify.embed_tags", True) is not False: + if metadata.get("spotify_track_id"): + source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] + if metadata.get("spotify_artist_id"): + source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] + if metadata.get("spotify_album_id"): + source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] + if cfg.get("itunes.embed_tags", True) is not False: + if metadata.get("itunes_track_id"): + source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] + if metadata.get("itunes_artist_id"): + source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] + if metadata.get("itunes_album_id"): + source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] + + track_title = metadata.get("title", "") + artist_name = metadata.get("album_artist", "") or metadata.get("artist", "") + track_info = get_import_track_info(context) + explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None + batch_artist_name = None + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + batch_artist_name = explicit_artist["name"] + elif isinstance(explicit_artist, str) and explicit_artist: + batch_artist_name = explicit_artist + + pp = { + "id_tags": source_ids, + "track_title": track_title, + "artist_name": artist_name, + "batch_artist_name": batch_artist_name, + "metadata": metadata, + "recording_mbid": None, + "artist_mbid": None, + "release_mbid": "", + "mb_genres": [], + "isrc": None, + "deezer_bpm": None, + "deezer_isrc": None, + "audiodb_mood": None, + "audiodb_style": None, + "audiodb_genre": None, + "tidal_isrc": None, + "tidal_copyright": None, + "qobuz_isrc": None, + "qobuz_copyright": None, + "qobuz_label": None, + "lastfm_tags": [], + "lastfm_url": None, + "genius_url": None, + "release_year": None, + } + + source_order = cfg.get("metadata_enhancement.post_process_order", None) + if not isinstance(source_order, list) or not source_order: + source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] + + db = _get_database(runtime) + + for source_name in source_order: + if source_name == "musicbrainz": + if cfg.get("musicbrainz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + mb_worker = getattr(runtime, "mb_worker", None) + mb_service = mb_worker.mb_service if mb_worker else None + if not mb_service: + continue + try: + result = mb_service.match_recording(track_title, artist_name) + if result and result.get("mbid"): + pp["recording_mbid"] = result["mbid"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] + details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) + if details: + isrcs = details.get("isrcs", []) + if isrcs: + pp["isrc"] = isrcs[0] + pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] + + track_artist_name = metadata.get("artist", "") or artist_name + if ", " in track_artist_name: + track_artist_name = track_artist_name.split(", ")[0] + artist_result = mb_service.match_artist(track_artist_name) + if artist_result and artist_result.get("mbid"): + pp["artist_mbid"] = artist_result["mbid"] + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] + + album_name_for_mb = metadata.get("album", "") + if album_name_for_mb: + artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() + rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) + rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) + with _MB_RELEASE_CACHE_LOCK: + cached = _MB_RELEASE_CACHE.get(rc_key_norm) + if cached is None: + cached = _MB_RELEASE_CACHE.get(rc_key_exact) + if cached is not None: + pp["release_mbid"] = cached + else: + try: + rc_result = mb_service.match_release(album_name_for_mb, artist_name) + pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" + except Exception: + pp["release_mbid"] = "" + _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] + _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] + if pp["release_mbid"]: + pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] + + if pp["release_mbid"]: + with _MB_RELEASE_DETAIL_CACHE_LOCK: + release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) + if release_detail is None: + release_detail = mb_service.mb_client.get_release( + pp["release_mbid"], + includes=["release-groups", "labels", "media", "artist-credits", "recordings"], + ) or {} + with _MB_RELEASE_DETAIL_CACHE_LOCK: + _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail + if release_detail: + rg = release_detail.get("release-group", {}) + if rg.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] + ac = release_detail.get("artist-credit", []) + if ac and isinstance(ac[0], dict): + aa = ac[0].get("artist", {}) + if aa.get("id"): + pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] + if rg.get("primary-type"): + pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] + if rg.get("first-release-date"): + pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] + if not pp["release_year"] and len(rg["first-release-date"]) >= 4: + year = rg["first-release-date"][:4] + if year.isdigit(): + pp["release_year"] = year + if release_detail.get("status"): + pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] + if release_detail.get("country"): + pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] + if release_detail.get("barcode"): + pp["id_tags"]["BARCODE"] = release_detail["barcode"] + media_list = release_detail.get("media", []) + if media_list: + fmt = media_list[0].get("format", "") + if fmt: + pp["id_tags"]["MEDIA"] = fmt + pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) + label_info = release_detail.get("label-info", []) + if label_info and isinstance(label_info[0], dict): + cat = label_info[0].get("catalog-number", "") + if cat: + pp["id_tags"]["CATALOGNUMBER"] = cat + text_rep = release_detail.get("text-representation", {}) + if isinstance(text_rep, dict) and text_rep.get("script"): + pp["id_tags"]["SCRIPT"] = text_rep["script"] + if release_detail.get("asin"): + pp["id_tags"]["ASIN"] = release_detail["asin"] + track_num = metadata.get("track_number") + disc_num = metadata.get("disc_number") or 1 + if track_num and media_list: + try: + track_num_int = int(track_num) + disc_num_int = int(disc_num) + for medium in media_list: + if medium.get("position", 1) == disc_num_int: + for mtrack in (medium.get("tracks") or medium.get("track-list", [])): + if mtrack.get("position") == track_num_int: + if mtrack.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] + release_recording = mtrack.get("recording", {}) + if release_recording.get("id"): + pp["recording_mbid"] = release_recording["id"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] + break + break + except (ValueError, TypeError): + pass + except Exception as exc: + logger_.error("MusicBrainz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "deezer": + if cfg.get("deezer.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + deezer_worker = getattr(runtime, "deezer_worker", None) + dz_client = deezer_worker.client if deezer_worker else None + if not dz_client: + continue + dz_result = dz_client.search_track(artist_name, track_title) + if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): + dz_track_id = dz_result["id"] + pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) + dz_artist_id = dz_result.get("artist", {}).get("id") + if dz_artist_id: + pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) + dz_details = dz_client.get_track_details(dz_track_id) + if dz_details: + bpm_val = dz_details.get("bpm") + if bpm_val and bpm_val > 0: + pp["deezer_bpm"] = bpm_val + dz_isrc = dz_details.get("isrc") + if dz_isrc: + pp["deezer_isrc"] = dz_isrc + if not pp["release_year"]: + dz_album = dz_result.get("album", {}) + dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" + if len(dz_release) >= 4 and dz_release[:4].isdigit(): + pp["release_year"] = dz_release[:4] + except Exception as exc: + logger_.error("Deezer lookup failed (non-fatal): %s", exc) + continue + + if source_name == "audiodb": + if cfg.get("audiodb.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + audiodb_worker = getattr(runtime, "audiodb_worker", None) + adb_client = audiodb_worker.client if audiodb_worker else None + if not adb_client: + continue + adb_result = adb_client.search_track(artist_name, track_title) + if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): + adb_track_id = adb_result.get("idTrack") + if adb_track_id: + pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) + adb_mb_track = adb_result.get("strMusicBrainzID") + if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track + pp["recording_mbid"] = adb_mb_track + adb_mb_artist = adb_result.get("strMusicBrainzArtistID") + if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist + pp["artist_mbid"] = adb_mb_artist + pp["audiodb_mood"] = adb_result.get("strMood") or None + pp["audiodb_style"] = adb_result.get("strStyle") or None + pp["audiodb_genre"] = adb_result.get("strGenre") or None + except Exception as exc: + logger_.error("AudioDB lookup failed (non-fatal): %s", exc) + continue + + if source_name == "tidal": + if cfg.get("tidal.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + tidal_client = getattr(runtime, "tidal_client", None) + if not (tidal_client and tidal_client.is_authenticated()): + continue + td_result = tidal_client.search_track(artist_name, track_title) + if td_result and _names_match(td_result.get("title", ""), track_title): + td_track_id = td_result.get("id") + if td_track_id: + pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) + td_artist = td_result.get("artist", {}) + if isinstance(td_artist, dict) and td_artist.get("id"): + pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) + if td_track_id: + td_details = tidal_client.get_track(str(td_track_id)) + if td_details: + pp["tidal_isrc"] = td_details.get("isrc") + td_copyright = td_details.get("copyright") + if isinstance(td_copyright, dict): + td_copyright = td_copyright.get("text", td_copyright.get("name", "")) + pp["tidal_copyright"] = td_copyright or None + if not pp["release_year"]: + td_album = td_result.get("album", {}) + td_release = "" + if isinstance(td_album, dict): + td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") + if len(td_release) >= 4 and td_release[:4].isdigit(): + pp["release_year"] = td_release[:4] + except Exception as exc: + logger_.error("Tidal lookup failed (non-fatal): %s", exc) + continue + + if source_name == "qobuz": + if cfg.get("qobuz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) + qz_client = qobuz_worker.client if qobuz_worker else None + if not (qz_client and qz_client.is_authenticated()): + continue + qz_result = qz_client.search_track(artist_name, track_title) + if qz_result: + qz_performer = qz_result.get("performer") or {} + if not isinstance(qz_performer, dict): + qz_performer = {} + qz_artist_name = qz_performer.get("name", "") + if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): + qz_track_id = qz_result.get("id") + if qz_track_id: + pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) + if qz_performer.get("id"): + pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) + qz_isrc = qz_result.get("isrc") + if isinstance(qz_isrc, dict): + qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) + if qz_isrc: + pp["qobuz_isrc"] = qz_isrc + qz_copyright = qz_result.get("copyright") + if isinstance(qz_copyright, dict): + qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) + if isinstance(qz_copyright, str): + pp["qobuz_copyright"] = qz_copyright + qz_album = qz_result.get("album", {}) + if isinstance(qz_album, dict): + qz_label_info = qz_album.get("label", {}) + if isinstance(qz_label_info, dict) and qz_label_info.get("name"): + pp["qobuz_label"] = qz_label_info["name"] + if not pp["release_year"]: + qz_release = str(qz_album.get("release_date_original", "") or "") + if not qz_release: + qz_ts = qz_album.get("released_at") + if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: + import datetime as _dt + qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) + if len(qz_release) >= 4 and qz_release[:4].isdigit(): + pp["release_year"] = qz_release[:4] + except Exception as exc: + logger_.error("Qobuz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "lastfm": + if cfg.get("lastfm.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + lastfm_worker = getattr(runtime, "lastfm_worker", None) + lf_client = lastfm_worker.client if lastfm_worker else None + if not lf_client: + continue + lf_result = lf_client.get_track_info(artist_name, track_title) + if lf_result: + lf_url = lf_result.get("url") + if lf_url: + pp["lastfm_url"] = lf_url + lf_toptags = lf_result.get("toptags", {}) + if isinstance(lf_toptags, dict): + tag_list = lf_toptags.get("tag", []) + if isinstance(tag_list, list): + pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] + elif isinstance(tag_list, dict) and tag_list.get("name"): + pp["lastfm_tags"] = [tag_list["name"]] + except Exception as exc: + logger_.error("Last.fm lookup failed (non-fatal): %s", exc) + continue + + if source_name == "genius": + if cfg.get("genius.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + import core.genius_client as _genius_module + + if time.time() < _genius_module._rate_limit_until: + logger_.info("Genius rate-limited, skipping (non-blocking)") + continue + genius_worker = getattr(runtime, "genius_worker", None) + g_client = genius_worker.client if genius_worker else None + if not g_client: + continue + g_result = g_client.search_song(artist_name, track_title) + if g_result: + g_id = g_result.get("id") + if g_id: + pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) + g_url = g_result.get("url") + if g_url: + pp["genius_url"] = g_url + except Exception as exc: + logger_.error("Genius lookup failed (non-fatal): %s", exc) + continue + + if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: + return + + filtered_tags: Dict[str, str] = {} + for tag_name, value in pp["id_tags"].items(): + config_path = tag_config.get(tag_name) + if config_path and not _tag_enabled(config_path): + continue + filtered_tags[tag_name] = value + + written = [] + id3_tag_map = { + "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), + "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), + "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), + "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), + "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), + "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), + "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), + "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), + "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), + "ORIGINALDATE": ("TDOR", None), + "MEDIA": ("TMED", None), + } + vorbis_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", + "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", + "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", + "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", + "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", + "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", + } + mp4_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", + "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", + "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", + "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", + "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", + "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", + "RELEASETYPE": "MusicBrainz Album Type", + "RELEASESTATUS": "MusicBrainz Album Status", + "RELEASECOUNTRY": "MusicBrainz Album Release Country", + } + + if isinstance(audio_file.tags, symbols.ID3): + for tag_name, value in filtered_tags.items(): + spec = id3_tag_map.get(tag_name) + if spec: + frame_type, desc = spec + if frame_type == "UFID": + audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) + written.append(f"UFID:{desc}") + elif frame_type == "TDOR": + audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) + written.append("TDOR") + elif frame_type == "TMED": + audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) + written.append("TMED") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) + written.append(f"TXXX:{desc}") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) + written.append(f"TXXX:{tag_name}") + elif _is_vorbis_like(audio_file, symbols): + for tag_name, value in filtered_tags.items(): + audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] + written.append(vorbis_tag_map.get(tag_name, tag_name)) + elif isinstance(audio_file, symbols.MP4): + for tag_name, value in filtered_tags.items(): + key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" + audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] + written.append(key) + + if written: + logger_.info("Embedded IDs: %s", ", ".join(written)) + + release_year = pp["release_year"] + needs_date_tag = bool(release_year and not metadata.get("date")) + if needs_date_tag: + metadata["date"] = release_year + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["date"] = [release_year] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9day"] = [release_year] + logger_.info("Date tag: %s", release_year) + + if _tag_enabled("deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: + bpm_int = int(pp["deezer_bpm"]) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["BPM"] = [str(bpm_int)] + elif isinstance(audio_file, symbols.MP4): + audio_file["tmpo"] = [bpm_int] + logger_.info("BPM: %s", bpm_int) + + if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["MOOD"] = [pp["audiodb_mood"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] + + if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["STYLE"] = [pp["audiodb_style"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] + + if _tag_enabled("metadata_enhancement.tags.genre_merge"): + enrichment_genres = [] + if _tag_enabled("musicbrainz.tags.genres"): + enrichment_genres += pp["mb_genres"] + if pp["audiodb_genre"] and _tag_enabled("audiodb.tags.genre"): + enrichment_genres.append(pp["audiodb_genre"]) + if _tag_enabled("lastfm.tags.genres"): + enrichment_genres += pp["lastfm_tags"] + if enrichment_genres: + from core.genre_filter import filter_genres as _filter_genres + + enrichment_genres = _filter_genres(enrichment_genres, cfg) + source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] + seen = set() + merged = [] + for genre in source_genres + enrichment_genres: + key = genre.strip().lower() + if key and key not in seen: + seen.add(key) + merged.append(genre.strip().title()) + if len(merged) >= 5: + break + if merged: + genre_string = ", ".join(merged) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENRE"] = [genre_string] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9gen"] = [genre_string] + logger_.info("Genres merged: %s", genre_string) + + isrc_candidates = [] + if pp["isrc"] and _tag_enabled("musicbrainz.tags.isrc"): + isrc_candidates.append(("MusicBrainz", pp["isrc"])) + if pp["deezer_isrc"] and _tag_enabled("deezer.tags.isrc"): + isrc_candidates.append(("Deezer", pp["deezer_isrc"])) + if pp["tidal_isrc"] and _tag_enabled("tidal.tags.isrc"): + isrc_candidates.append(("Tidal", pp["tidal_isrc"])) + if pp["qobuz_isrc"] and _tag_enabled("qobuz.tags.isrc"): + isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) + if isrc_candidates: + isrc_source, final_isrc = isrc_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["ISRC"] = [final_isrc] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] + logger_.info("ISRC (%s): %s", isrc_source, final_isrc) + + copyright_candidates = [] + if pp["tidal_copyright"] and _tag_enabled("tidal.tags.copyright"): + copyright_candidates.append(("Tidal", pp["tidal_copyright"])) + if pp["qobuz_copyright"] and _tag_enabled("qobuz.tags.copyright"): + copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) + if copyright_candidates: + copyright_source, final_copyright = copyright_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["COPYRIGHT"] = [final_copyright] + elif isinstance(audio_file, symbols.MP4): + audio_file["cprt"] = [final_copyright] + logger_.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) + + if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LABEL"] = [pp["qobuz_label"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] + + if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LASTFM_URL"] = [pp["lastfm_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] + + if _tag_enabled("genius.tags.url") and pp["genius_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENIUS_URL"] = [pp["genius_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] + + release_id = pp["release_mbid"] + if release_id: + metadata["musicbrainz_release_id"] = release_id + if db is not None: + try: + album_name_for_db = metadata.get("album", "") + album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") + if album_name_for_db and album_artist_for_db: + conn = db._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE albums SET year = ? + WHERE (year IS NULL OR year = 0) + AND id IN ( + SELECT al.id FROM albums al + JOIN artists ar ON ar.id = al.artist_id + WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) + ) + """, + (int(release_year), album_name_for_db, album_artist_for_db), + ) + if cursor.rowcount > 0: + conn.commit() + logger_.info("Updated album year to %s in database", release_year) + else: + conn.rollback() + finally: + conn.close() + except Exception as exc: + logger_.error("Could not update album year in DB: %s", exc) + + except Exception as exc: + logger_.error("Error embedding source IDs (non-fatal): %s", exc) + + +def download_cover_art(album_info: dict, target_dir: str, context: dict = None, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.cover_art_download", True) is False: + return + + try: + cover_path = os.path.join(target_dir, "cover.jpg") + album_info = album_info or {} + release_mbid = album_info.get("musicbrainz_release_id") + prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False) + + if os.path.exists(cover_path): + if release_mbid and prefer_caa: + try: + existing_size = os.path.getsize(cover_path) + if existing_size > 200_000: + return + is_upgrade = True + except Exception: + return + else: + return + else: + is_upgrade = False + + image_data = None + if release_mbid and prefer_caa: + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if is_upgrade and not image_data: + logger_.error("CAA upgrade failed - keeping existing cover.jpg") + return + + if not image_data: + art_url = album_info.get("album_image_url") + if not art_url and context: + album_ctx = get_import_context_album(context) + art_url = album_ctx.get("image_url") + if not art_url and album_ctx.get("images"): + images = album_ctx.get("images", []) + if images and isinstance(images[0], dict): + art_url = images[0].get("url", "") + if art_url: + logger_.info("Using cover art URL from album context") + if art_url and "i.scdn.co" in art_url: + try: + from core.spotify_client import _upgrade_spotify_image_url + + art_url = _upgrade_spotify_image_url(art_url) + except Exception: + pass + elif art_url and "mzstatic.com" in art_url: + import re as _re + + art_url = _re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) + if not art_url: + logger_.warning("No cover art URL available for download.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + + if not image_data: + return + + with open(cover_path, "wb") as handle: + handle.write(image_data) + logger_.info("Cover art downloaded to: %s", cover_path) + except Exception as exc: + logger_.error("Error downloading cover.jpg: %s", exc) + + +def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: + return False + + try: + from core.lyrics_client import lyrics_client + + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_context = get_import_context_album(context) + track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) + + if isinstance(artist, dict): + artist_name = artist.get("name", "Unknown Artist") + elif hasattr(artist, "name"): + artist_name = artist.name + else: + artist_name = str(artist) if artist else "Unknown Artist" + + album_name = None + duration_seconds = None + if album_info and album_info.get("is_album"): + album_name = ( + get_import_clean_album(context, album_info=album_info, default="") + or album_info.get("album_name") + or album_context.get("name") + ) + + if original_search.get("duration_ms"): + duration_seconds = int(original_search["duration_ms"] / 1000) + + success = lyrics_client.create_lrc_file( + audio_file_path=file_path, + track_name=track_name, + artist_name=artist_name, + album_name=album_name, + duration_seconds=duration_seconds, + ) + + if success: + logger_.info("LRC file generated for: %s", track_name) + else: + logger_.warning("No lyrics found for: %s", track_name) + return success + except Exception as exc: + logger_.error("Error generating LRC file for %s: %s", file_path, exc) + return False + + +def wipe_source_tags(file_path: str, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + _ = cfg # keep signature parallel with other helpers + + try: + _strip_all_non_audio_tags(file_path, runtime=runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return False + + audio = symbols.File(file_path) + if audio is None: + return False + if hasattr(audio, "clear_pictures"): + audio.clear_pictures() + if audio.tags is not None: + tag_count = len(audio.tags) + audio.tags.clear() + else: + audio.add_tags() + tag_count = 0 + _save_audio_file(audio, symbols) + if tag_count > 0: + logger_.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path)) + return True + except Exception as exc: + logger_.error("[Tag Wipe] Failed (non-fatal): %s", exc) + return False + + +def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.enabled", True) is False: + logger_.warning("Metadata enhancement disabled in config.") + return True + + if album_info is None: + album_info = {} + + symbols = _get_mutagen_symbols(runtime) + if not symbols: + logger_.error("Mutagen is unavailable, cannot enhance metadata.") + return False + + file_lock = _get_file_lock(file_path) + with file_lock: + logger_.info("Enhancing metadata for: %s", os.path.basename(file_path)) + try: + _strip_all_non_audio_tags(file_path, runtime=runtime) + audio_file = symbols.File(file_path) + if audio_file is None: + logger_.error("Could not load audio file with Mutagen: %s", file_path) + return False + + if hasattr(audio_file, "clear_pictures"): + audio_file.clear_pictures() + + if audio_file.tags is not None: + if len(audio_file.tags) > 0: + tag_keys = list(audio_file.tags.keys())[:15] + logger_.info("Clearing %s existing tags: %s", len(audio_file.tags), ", ".join(str(k) for k in tag_keys)) + audio_file.tags.clear() + else: + audio_file.add_tags() + + _save_audio_file(audio_file, symbols) + + metadata = extract_source_metadata(context, artist, album_info, runtime=runtime) + if not metadata: + logger_.error("Could not extract source metadata, saving with cleared tags.") + _save_audio_file(audio_file, symbols) + return True + + track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}" + write_multi = cfg.get("metadata_enhancement.tags.write_multi_artist", False) + artists_list = metadata.get("_artists_list", []) + + if isinstance(audio_file.tags, symbols.ID3): + if metadata.get("title"): + audio_file.tags.add(symbols.TIT2(encoding=3, text=[metadata["title"]])) + if metadata.get("artist"): + audio_file.tags.add(symbols.TPE1(encoding=3, text=[metadata["artist"]])) + if write_multi and len(artists_list) > 1: + audio_file.tags.add(symbols.TPE1(encoding=3, text=artists_list)) + if metadata.get("album_artist"): + audio_file.tags.add(symbols.TPE2(encoding=3, text=[metadata["album_artist"]])) + if metadata.get("album"): + audio_file.tags.add(symbols.TALB(encoding=3, text=[metadata["album"]])) + if metadata.get("date"): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[metadata["date"]])) + if metadata.get("genre"): + audio_file.tags.add(symbols.TCON(encoding=3, text=[metadata["genre"]])) + audio_file.tags.add(symbols.TRCK(encoding=3, text=[track_num_str])) + if metadata.get("disc_number"): + audio_file.tags.add(symbols.TPOS(encoding=3, text=[str(metadata["disc_number"])])) + elif _is_vorbis_like(audio_file, symbols): + if metadata.get("title"): + audio_file["title"] = [metadata["title"]] + if metadata.get("artist"): + audio_file["artist"] = [metadata["artist"]] + if write_multi and len(artists_list) > 1: + audio_file["artists"] = artists_list + if metadata.get("album_artist"): + audio_file["albumartist"] = [metadata["album_artist"]] + if metadata.get("album"): + audio_file["album"] = [metadata["album"]] + if metadata.get("date"): + audio_file["date"] = [metadata["date"]] + if metadata.get("genre"): + audio_file["genre"] = [metadata["genre"]] + audio_file["tracknumber"] = [track_num_str] + if metadata.get("disc_number"): + audio_file["discnumber"] = [str(metadata["disc_number"])] + elif isinstance(audio_file, symbols.MP4): + if metadata.get("title"): + audio_file["\xa9nam"] = [metadata["title"]] + if metadata.get("artist"): + audio_file["\xa9ART"] = artists_list if (write_multi and len(artists_list) > 1) else [metadata["artist"]] + if metadata.get("album_artist"): + audio_file["aART"] = [metadata["album_artist"]] + if metadata.get("album"): + audio_file["\xa9alb"] = [metadata["album"]] + if metadata.get("date"): + audio_file["\xa9day"] = [metadata["date"]] + if metadata.get("genre"): + audio_file["\xa9gen"] = [metadata["genre"]] + audio_file["trkn"] = [(metadata.get("track_number", 1), metadata.get("total_tracks", 1))] + if metadata.get("disc_number"): + audio_file["disk"] = [(metadata["disc_number"], 0)] + + embed_source_ids(audio_file, metadata, context, runtime=runtime) + + if album_info is not None and metadata.get("musicbrainz_release_id"): + album_info["musicbrainz_release_id"] = metadata["musicbrainz_release_id"] + + if cfg.get("metadata_enhancement.embed_album_art", True): + embed_album_art_metadata(audio_file, metadata, runtime=runtime) + + quality = context.get("_audio_quality", "") + if quality and cfg.get("metadata_enhancement.tags.quality_tag", True) is not False: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="QUALITY", text=[quality])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["quality"] = [quality] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:QUALITY"] = [symbols.MP4FreeForm(quality.encode("utf-8"))] + + _save_audio_file(audio_file, symbols) + + verified = _verify_metadata_written(file_path, runtime=runtime) + if verified: + logger_.info("Metadata enhanced successfully.") + else: + logger_.info("Metadata saved but verification found issues (see above).") + return True + except Exception as exc: + import traceback + + logger_.error("Error enhancing metadata for %s: %s", file_path, exc) + logger_.error("[Metadata Debug] Exception type: %s", type(exc).__name__) + logger_.info("[Metadata Debug] File exists: %s", os.path.exists(file_path)) + logger_.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None") + logger_.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None") + logger_.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc()) + return False + + +def _normalize_album_cache_key(album_name: str) -> str: + result = _EDITION_PAREN_RE.sub("", album_name or "") + result = _EDITION_BARE_RE.sub("", result) + return result.lower().strip() diff --git a/core/metadata_service.py b/core/metadata_service.py index e2636d0f..861f0b08 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -719,6 +719,23 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti if not isinstance(images, list): images = list(images) if images else [] + artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + if not artists and artist_name: + artists = [{'name': artist_name}] + + primary_artist = artists[0] if artists else {} + resolved_artist_name = ( + _extract_lookup_value(primary_artist, 'name', default='') + or artist_name + or _extract_lookup_value(album_data, 'artist_name', 'artist', default='') + or '' + ) + resolved_artist_id = str( + _extract_lookup_value(primary_artist, 'id', default='') + or _extract_lookup_value(album_data, 'artist_id', default='') + or '' + ).strip() + image_url = None if images: image_url = _extract_lookup_value(images[0], 'url') @@ -728,12 +745,15 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti return { 'id': _extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', 'release_id', default=album_id) or album_id, 'name': _extract_lookup_value(album_data, 'name', 'title', default=album_name or album_id) or album_name or album_id, + 'artist': resolved_artist_name or '', + 'artist_name': resolved_artist_name or '', + 'artist_id': resolved_artist_id, + 'artists': artists, 'image_url': image_url, 'images': images, 'release_date': _extract_lookup_value(album_data, 'release_date', default='') or '', 'album_type': _extract_lookup_value(album_data, 'album_type', default='album') or 'album', 'total_tracks': _extract_lookup_value(album_data, 'total_tracks', 'track_count', default=0) or 0, - 'artist_name': artist_name or _extract_lookup_value(album_data, 'artist_name', default='') or '', } @@ -754,6 +774,8 @@ def _build_album_track_entry(track_item: Any, album_info: Dict[str, Any], source 'external_urls': _extract_lookup_value(track_item, 'external_urls', default={}) or {}, 'uri': _extract_lookup_value(track_item, 'uri', default='') or '', 'album': album_info, + 'source': source, + 'provider': source, '_source': source, } @@ -767,6 +789,9 @@ def _build_album_tracks_payload( artist_name: str = '', ) -> Dict[str, Any]: album_info = _build_album_info(album_data, album_id, album_name=album_name, artist_name=artist_name) + album_info['source'] = source + album_info['_source'] = source + album_info['provider'] = source track_items = _extract_album_track_items(album_data, tracks_data) tracks = [_build_album_track_entry(track, album_info, source) for track in track_items] @@ -778,6 +803,369 @@ def _build_album_tracks_payload( } +def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + normalized: List[Dict[str, Any]] = [] + for artist in artists: + if isinstance(artist, dict): + name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or '' + artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or '' + entry: Dict[str, Any] = {} + if name: + entry['name'] = str(name) + if artist_id: + entry['id'] = str(artist_id) + genres = _extract_lookup_value(artist, 'genres', default=None) + if genres is not None: + entry['genres'] = genres + if entry: + normalized.append(entry) + continue + + name = str(artist).strip() + if name: + normalized.append({'name': name}) + + return normalized + + +def _build_single_import_context_payload( + track_data: Any, + source: Optional[str], + source_priority: List[str], + requested_title: str = '', + requested_artist: str = '', +) -> Dict[str, Any]: + album_data = _extract_lookup_value(track_data, 'album', default=None) + + track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '') + track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track' + track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[])) + if not track_artists and requested_artist: + track_artists = [{'name': requested_artist}] + + primary_track_artist = track_artists[0] if track_artists else {} + primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist' + primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '') + + album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or '' + album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '') + release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '') + album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album') + total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0) + album_images: List[Dict[str, Any]] = [] + album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[])) + + if isinstance(album_data, dict): + album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name + album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) + release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) + album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) + total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) + album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] + if not album_image_url: + album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + elif album_data: + album_name = album_name or str(album_data) + + if not album_artists and primary_artist_name: + album_artists = [{'name': primary_artist_name}] + + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + + track_info = { + 'id': track_id, + 'name': track_name, + 'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1), + 'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1), + 'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0), + 'artists': track_artists or [{'name': primary_artist_name}], + 'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''), + 'album': album_name, + 'album_id': album_id, + 'album_type': album_type, + 'release_date': release_date, + '_source': source or '', + } + + album_payload = { + 'id': album_id, + 'name': album_name, + 'release_date': release_date, + 'total_tracks': total_tracks or 1, + 'album_type': album_type, + 'image_url': album_image_url, + 'images': album_images, + 'artists': album_artists, + '_source': source or '', + } + + artist_payload = { + 'id': primary_artist_id, + 'name': primary_artist_name, + 'genres': [], + '_source': source or '', + } + + original_search = { + 'title': track_name, + 'artist': primary_artist_name, + 'album': album_name, + 'track_number': track_info['track_number'], + 'disc_number': track_info['disc_number'], + 'clean_title': track_name, + 'clean_album': album_name, + 'clean_artist': primary_artist_name, + 'artists': track_info['artists'], + 'duration_ms': track_info['duration_ms'], + 'id': track_id, + '_source': source or '', + } + + return { + 'success': bool(track_id or track_name != requested_title or album_name), + 'source': source, + 'source_priority': source_priority, + 'context': { + 'artist': artist_payload, + 'album': album_payload, + 'track_info': track_info, + 'original_search_result': original_search, + 'is_album_download': False, + 'has_clean_metadata': bool(track_id), + 'has_full_metadata': bool(track_id), + 'source': source, + 'source_priority': source_priority, + }, + } + + +def _build_single_import_fallback_context( + requested_title: str, + requested_artist: str, + source_priority: List[str], +) -> Dict[str, Any]: + artist_name = requested_artist or 'Unknown Artist' + title = requested_title or 'Unknown Track' + return { + 'success': False, + 'source': None, + 'source_priority': source_priority, + 'context': { + 'artist': { + 'id': '', + 'name': artist_name, + 'genres': [], + '_source': '', + }, + 'album': { + 'id': '', + 'name': '', + 'release_date': '', + 'total_tracks': 1, + 'album_type': 'album', + 'image_url': '', + 'images': [], + 'artists': [], + '_source': '', + }, + 'track_info': { + 'id': '', + 'name': title, + 'track_number': 1, + 'disc_number': 1, + 'duration_ms': 0, + 'artists': [{'name': artist_name}], + 'uri': '', + 'album': '', + 'album_id': '', + 'album_type': 'album', + 'release_date': '', + '_source': '', + }, + 'original_search_result': { + 'title': title, + 'artist': artist_name, + 'album': '', + 'track_number': 1, + 'disc_number': 1, + 'clean_title': title, + 'clean_album': '', + 'clean_artist': artist_name, + 'artists': [{'name': artist_name}], + 'duration_ms': 0, + 'id': '', + '_source': '', + }, + 'is_album_download': False, + 'has_clean_metadata': False, + 'has_full_metadata': False, + 'source': None, + 'source_priority': source_priority, + }, + } + + +def _build_track_search_query(source: str, title: str, artist: str) -> str: + base_query = " ".join(part for part in (title, artist) if part).strip() + if source == 'deezer' and title: + if artist: + return f'artist:"{artist}" track:"{title}"' + return f'track:"{title}"' + return base_query or title or artist + + +def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]: + if not search_results: + return None + + target_title = str(title or '').strip().lower() + target_artist = str(artist or '').strip().lower() + + for candidate in search_results: + candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower() + if candidate_title != target_title: + continue + + if not target_artist: + return candidate + + candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[])) + candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else '' + if candidate_artist_name == target_artist: + return candidate + + return search_results[0] + + +def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: + if not client or not hasattr(client, 'search_tracks'): + return [] + + try: + kwargs = {'limit': limit} + if source == 'spotify': + kwargs['allow_fallback'] = False + return client.search_tracks(query, **kwargs) or [] + except Exception as exc: + logger.debug("Could not search %s for %s: %s", source, query, exc) + return [] + + +def get_single_track_import_context( + title: str, + artist: str = '', + override_id: Optional[str] = None, + override_source: str = 'spotify', + source_override: Optional[str] = None, +) -> Dict[str, Any]: + """Build an import context for singles using source-priority metadata lookup.""" + options = MetadataLookupOptions(source_override=source_override, allow_fallback=True) + source_priority = _get_source_chain_for_lookup(options) + title = (title or '').strip() + artist = (artist or '').strip() + + if override_id: + chosen_source = (override_source or 'spotify').strip().lower() or 'spotify' + client = get_client_for_source(chosen_source) + if client and hasattr(client, 'get_track_details'): + try: + track_data = client.get_track_details(str(override_id)) + if track_data: + payload = _build_single_import_context_payload( + track_data, + chosen_source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + except Exception as exc: + logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc) + + for source in source_priority: + client = get_client_for_source(source) + if not client: + continue + + search_query = _build_track_search_query(source, title, artist) + if not search_query: + continue + + search_results = _search_tracks_for_source(source, client, search_query, limit=5) + if not search_results and search_query != title: + search_results = _search_tracks_for_source(source, client, title, limit=5) + if not search_results and artist and search_query != artist: + search_results = _search_tracks_for_source(source, client, artist, limit=5) + + if not search_results: + continue + + best_match = _pick_best_track_match(search_results, title or search_query, artist) + if not best_match: + continue + + resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '') + resolved_data = best_match + if resolved_track_id and hasattr(client, 'get_track_details'): + try: + detailed = client.get_track_details(resolved_track_id) + if detailed: + resolved_data = detailed + except Exception as exc: + logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc) + + payload = _build_single_import_context_payload( + resolved_data, + source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + + return _build_single_import_fallback_context(title, artist, source_priority) + + def resolve_album_reference( album_id: str, preferred_source: Optional[str] = None, diff --git a/core/soulseek_client.py b/core/soulseek_client.py index 9b64ada6..e859fd4e 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -8,6 +8,7 @@ import time from pathlib import Path from utils.logging_config import get_logger from config.settings import config_manager +from core.import_filename import parse_filename_metadata logger = get_logger("soulseek_client") @@ -87,60 +88,17 @@ class TrackResult(SearchResult): def _parse_filename_metadata(self): """Extract artist, title, album from filename patterns""" - import re - import os - - # Get just the filename without extension and path - base_name = os.path.splitext(os.path.basename(self.filename))[0] - - # Common patterns for track naming - patterns = [ - r'^(\d+)\s*[-\.]\s*(.+?)\s*[-–]\s*(.+)$', # "01 - Artist - Title" or "01. Artist - Title" - r'^(.+?)\s*[-–]\s*(.+)$', # "Artist - Title" - r'^(\d+)\s*[-\.]\s*(.+)$', # "01 - Title" or "01. Title" - ] - - for pattern in patterns: - match = re.match(pattern, base_name) - if match: - groups = match.groups() - if len(groups) == 3: # Track number, artist, title - try: - self.track_number = int(groups[0]) - self.artist = self.artist or groups[1].strip() - self.title = self.title or groups[2].strip() - except ValueError: - # First group might not be a number - self.artist = self.artist or groups[0].strip() - self.title = self.title or f"{groups[1]} - {groups[2]}".strip() - elif len(groups) == 2: - if groups[0].isdigit(): # Track number and title - try: - self.track_number = int(groups[0]) - self.title = self.title or groups[1].strip() - except ValueError: - pass - else: # Artist and title - self.artist = self.artist or groups[0].strip() - self.title = self.title or groups[1].strip() - break - - # Fallback: use filename as title if nothing was extracted - if not self.title: - self.title = base_name - - # Try to extract album from directory path - if not self.album and '/' in self.filename: - path_parts = self.filename.split('/') - if len(path_parts) >= 2: - # Look for album-like directory names - for part in reversed(path_parts[:-1]): # Exclude filename - if part and not part.startswith('@'): # Skip system directories - # Clean up common patterns - cleaned = re.sub(r'^\d+\s*[-\.]\s*', '', part) # Remove leading numbers - if len(cleaned) > 3: # Must be substantial - self.album = cleaned - break + parsed = parse_filename_metadata(self.filename) + if not self.artist and parsed.get("artist"): + self.artist = parsed["artist"] + if not self.title and parsed.get("title"): + self.title = parsed["title"] + if not self.album and parsed.get("album"): + self.album = parsed["album"] + if self.track_number is None: + track_number = parsed.get("track_number") + if track_number is not None: + self.track_number = track_number @dataclass class AlbumResult: @@ -1829,4 +1787,4 @@ class SoulseekClient: def __del__(self): # No persistent session to clean up - pass \ No newline at end of file + pass diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index a0cdd66b..c1d9aa86 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -2955,9 +2955,6 @@ class WatchlistScanner: } track_data = { - 'spotify_track_id': track['id'], - 'spotify_album_id': album_data['id'], - 'spotify_artist_id': artist.spotify_artist_id, 'track_name': track['name'], 'artist_name': artist.artist_name, 'album_name': album_data.get('name', 'Unknown Album'), diff --git a/tests/test_downloads_pagination.py b/tests/test_downloads_pagination.py index af2c7b9a..f85dd96d 100644 --- a/tests/test_downloads_pagination.py +++ b/tests/test_downloads_pagination.py @@ -7,7 +7,6 @@ and `status` query params and includes a `total` count. """ import sys -import threading import types from unittest.mock import patch @@ -45,6 +44,7 @@ _install_flask_limiter_stub() from flask import Flask, Blueprint # noqa: E402 from api import downloads as downloads_mod # noqa: E402 +import core.import_runtime_state as runtime_state # noqa: E402 def _make_task(status="downloading", when=None): @@ -63,11 +63,10 @@ def _make_task(status="downloading", when=None): def _make_app_with_tasks(tasks_dict): """Create a minimal Flask app with the downloads blueprint mounted and - a fake `web_server` module exposing the given download_tasks dict.""" - fake_ws = types.ModuleType("web_server") - fake_ws.download_tasks = tasks_dict - fake_ws.tasks_lock = threading.RLock() - sys.modules["web_server"] = fake_ws + the shared import runtime state populated with the given download_tasks dict.""" + original_tasks = dict(runtime_state.download_tasks) + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(tasks_dict) # Bypass API key auth for tests. def _passthrough(f): @@ -83,6 +82,7 @@ def _make_app_with_tasks(tasks_dict): downloads_mod.register_routes(bp) app.register_blueprint(bp) + app._original_download_tasks = original_tasks return app @@ -96,8 +96,12 @@ def client(): for i in range(25) } app = _make_app_with_tasks(tasks) - with app.test_client() as c: - yield c + try: + with app.test_client() as c: + yield c + finally: + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(app._original_download_tasks) def test_default_limit_applied(client): diff --git a/tests/test_import_album.py b/tests/test_import_album.py new file mode 100644 index 00000000..81ab6fb6 --- /dev/null +++ b/tests/test_import_album.py @@ -0,0 +1,186 @@ +import core.import_album as import_album + + +class _FakeEngine: + def normalize_string(self, text): + return str(text or "").strip().lower() + + def similarity_score(self, left, right): + return 1.0 if left == right else 0.0 + + +def test_resolve_album_artist_context_uses_provider_genres(monkeypatch): + class _FakeClient: + def get_artist(self, artist_id): + assert artist_id == "artist-1" + return {"genres": ["rock", "indie"]} + + monkeypatch.setattr(import_album, "get_client_for_source", lambda source: _FakeClient() if source == "spotify" else None) + + context = import_album.resolve_album_artist_context( + { + "id": "album-1", + "name": "Album One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + }, + source="spotify", + ) + + assert context == { + "id": "artist-1", + "name": "Artist One", + "genres": ["rock", "indie"], + "source": "spotify", + } + + +def test_build_album_import_context_is_neutral(): + context = import_album.build_album_import_context( + { + "id": "album-1", + "name": "Album One", + "artist": "Artist One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + "source": "deezer", + }, + { + "id": "track-1", + "name": "Song One", + "track_number": 3, + "disc_number": 2, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "deezer:track:track-1", + "album": "Album One", + "source": "deezer", + }, + artist_context={ + "id": "artist-1", + "name": "Artist One", + "genres": ["rock"], + "source": "deezer", + }, + total_discs=2, + source="deezer", + ) + + assert context["artist"]["name"] == "Artist One" + assert context["artist"]["genres"] == ["rock"] + assert context["album"]["name"] == "Album One" + assert context["album"]["total_discs"] == 2 + assert context["track_info"]["name"] == "Song One" + assert context["track_info"]["track_number"] == 3 + assert context["original_search_result"]["clean_title"] == "Song One" + assert context["source"] == "deezer" + assert "spotify_artist" not in context + assert "spotify_album" not in context + + +def test_build_album_import_match_payload_uses_generic_track_keys(monkeypatch, tmp_path): + staging_root = tmp_path / "Staging" + staging_root.mkdir() + (staging_root / "Song One.flac").write_text("fake") + + monkeypatch.setattr(import_album, "get_staging_path", lambda: str(staging_root)) + monkeypatch.setattr(import_album, "_get_matching_engine", lambda: _FakeEngine()) + monkeypatch.setattr( + import_album, + "read_staging_file_metadata", + lambda file_path, filename=None: { + "title": "Song One", + "artist": "Artist One", + "albumartist": "Artist One", + "album": "Album One", + "track_number": 1, + "disc_number": 1, + }, + ) + monkeypatch.setattr( + import_album, + "get_artist_album_tracks", + lambda album_id, artist_name="", album_name="", source=None: { + "success": True, + "album": { + "id": album_id, + "name": "Album One", + "artist": "Artist One", + "artist_name": "Artist One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + "release_date": "2024-01-01", + "total_tracks": 1, + "total_discs": 1, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + "images": [{"url": "https://img.example/album.jpg"}], + "source": "spotify", + }, + "tracks": [ + { + "id": "track-1", + "name": "Song One", + "track_number": 1, + "disc_number": 1, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "spotify:track:track-1", + "album": { + "id": album_id, + "name": "Album One", + "artist": "Artist One", + }, + "source": "spotify", + } + ], + "source": "spotify", + "source_priority": ["spotify"], + "resolved_album_id": album_id, + }, + ) + + result = import_album.build_album_import_match_payload( + "album-1", + album_name="Album One", + album_artist="Artist One", + source="spotify", + ) + + assert result["success"] is True + assert result["album"]["artist"] == "Artist One" + assert result["source"] == "spotify" + assert result["matches"] == [ + { + "track": { + "id": "track-1", + "name": "Song One", + "track_number": 1, + "disc_number": 1, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "spotify:track:track-1", + "album": { + "id": "album-1", + "name": "Album One", + "artist": "Artist One", + }, + "source": "spotify", + }, + "staging_file": { + "filename": "Song One.flac", + "full_path": str(staging_root / "Song One.flac"), + "title": "Song One", + "artist": "Artist One", + "album": "Album One", + "albumartist": "Artist One", + "track_number": 1, + "disc_number": 1, + }, + "confidence": 1.0, + } + ] diff --git a/tests/test_import_context.py b/tests/test_import_context.py new file mode 100644 index 00000000..f91acf19 --- /dev/null +++ b/tests/test_import_context.py @@ -0,0 +1,202 @@ +import pytest + +from core.import_context import ( + build_import_album_info, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_has_clean_metadata, + get_import_has_full_metadata, + get_import_source, + get_import_source_ids, + get_library_source_id_columns, + get_source_tag_names, + normalize_import_context, +) + + +def test_normalize_import_context_promotes_neutral_fields_without_legacy_aliases(): + context = { + "source": "deezer", + "spotify_artist": {"name": "Artist One", "id": "artist-1"}, + "spotify_album": { + "name": "Album One", + "id": "album-1", + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 3, + "disc_number": 2, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "spotify_clean_title": "Song One", + "spotify_clean_album": "Album One", + "spotify_clean_artist": "Artist One", + }, + "has_clean_spotify_data": True, + "has_full_spotify_metadata": False, + } + + normalized = normalize_import_context(context) + + assert normalized["artist"]["name"] == "Artist One" + assert normalized["album"]["name"] == "Album One" + assert "spotify_artist" not in normalized + assert "spotify_album" not in normalized + assert normalized["original_search_result"]["clean_title"] == "Song One" + assert normalized["original_search_result"]["clean_album"] == "Album One" + assert normalized["original_search_result"]["clean_artist"] == "Artist One" + assert "spotify_clean_title" not in normalized["original_search_result"] + assert "spotify_clean_album" not in normalized["original_search_result"] + assert "spotify_clean_artist" not in normalized["original_search_result"] + assert get_import_clean_title(normalized) == "Song One" + assert get_import_clean_album(normalized) == "Album One" + assert get_import_clean_artist(normalized) == "Artist One" + assert get_import_source(normalized) == "deezer" + assert get_import_has_clean_metadata(normalized) is True + assert get_import_has_full_metadata(normalized) is False + + +def test_neutral_import_context_helpers_work_without_legacy_aliases(): + context = { + "source": "deezer", + "artist": {"name": "Artist One", "id": "artist-1"}, + "album": { + "name": "Album One", + "id": "album-1", + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 3, + "disc_number": 2, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "artist": "Artist One", + "album": "Album One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + }, + "has_clean_metadata": True, + "has_full_metadata": True, + } + + assert get_import_clean_title(context) == "Song One" + assert get_import_clean_album(context) == "Album One" + assert get_import_clean_artist(context) == "Artist One" + assert get_import_source(context) == "deezer" + + normalized = normalize_import_context(context) + assert normalized["artist"]["name"] == "Artist One" + assert normalized["album"]["name"] == "Album One" + assert normalized["original_search_result"]["clean_title"] == "Song One" + assert "spotify_artist" not in normalized + assert "spotify_album" not in normalized + assert get_import_has_clean_metadata(normalized) is True + assert get_import_has_full_metadata(normalized) is True + + +@pytest.mark.parametrize( + "source,expected_tags,expected_columns", + [ + ( + "spotify", + {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"}, + {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"}, + ), + ( + "itunes", + {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"}, + {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"}, + ), + ( + "deezer", + {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None}, + {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"}, + ), + ( + "hydrabase", + {"track": None, "artist": None, "album": None}, + {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"}, + ), + ( + "discogs", + {"track": None, "artist": None, "album": None}, + {"artist": "discogs_id", "album": "discogs_id", "track": None}, + ), + ], +) +def test_source_tag_and_library_column_mappings(source, expected_tags, expected_columns): + assert get_source_tag_names(source) == expected_tags + assert get_library_source_id_columns(source) == expected_columns + + +def test_get_import_source_ids_prefers_nested_source_specific_ids(): + context = normalize_import_context( + { + "source": "deezer", + "artist": {"deezer_id": "deezer-artist-1"}, + "album": {"deezer_id": "deezer-album-1"}, + "track_info": {"deezer_id": "deezer-track-1"}, + "original_search_result": {"source_artist_id": "deezer-artist-1"}, + } + ) + + assert get_import_source_ids(context) == { + "track_id": "deezer-track-1", + "artist_id": "deezer-artist-1", + "album_id": "deezer-album-1", + } + + +def test_build_import_album_info_uses_normalized_album_context(): + context = normalize_import_context( + { + "source": "deezer", + "artist": {"name": "Artist One"}, + "album": { + "name": "Album One", + "image_url": "https://img.example/album.jpg", + "release_date": "2024-05-01", + "total_tracks": 8, + "album_type": "album", + }, + "track_info": { + "name": "Song One", + "track_number": 4, + "disc_number": 2, + "duration_ms": 240000, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "album": "Album One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + }, + } + ) + + album_info = build_import_album_info(context) + + assert album_info["is_album"] is True + assert album_info["album_name"] == "Album One" + assert album_info["track_number"] == 4 + assert album_info["disc_number"] == 2 + assert album_info["clean_track_name"] == "Song One" + assert album_info["album_image_url"] == "https://img.example/album.jpg" + assert album_info["source"] == "deezer" diff --git a/tests/test_import_file_ops.py b/tests/test_import_file_ops.py new file mode 100644 index 00000000..e9d72901 --- /dev/null +++ b/tests/test_import_file_ops.py @@ -0,0 +1,92 @@ +import sys +import types + +from core.import_file_ops import ( + cleanup_empty_directories, + extract_track_number_from_filename, + safe_move_file, + read_staging_file_metadata, +) + + +def test_extract_track_number_from_filename_handles_common_patterns(): + assert extract_track_number_from_filename("01 - Song.mp3") == 1 + assert extract_track_number_from_filename("1-03 - Song.mp3") == 3 + assert extract_track_number_from_filename("Artist - Song.mp3") == 1 + + +def test_safe_move_file_replaces_existing_destination(tmp_path): + src = tmp_path / "source.flac" + dst_dir = tmp_path / "dest" + dst_dir.mkdir() + dst = dst_dir / "track.flac" + + src.write_text("new") + dst.write_text("old") + + safe_move_file(src, dst) + + assert not src.exists() + assert dst.read_text() == "new" + + +def test_cleanup_empty_directories_removes_nested_empty_paths(tmp_path): + download_root = tmp_path / "downloads" + nested_dir = download_root / "Artist" / "Album" + nested_dir.mkdir(parents=True) + moved_file_path = nested_dir / "track.flac" + + cleanup_empty_directories(str(download_root), str(moved_file_path)) + + assert not nested_dir.exists() + assert not (download_root / "Artist").exists() + assert download_root.exists() + + +def test_read_staging_file_metadata_reads_tags(monkeypatch, tmp_path): + file_path = tmp_path / "Song One.flac" + file_path.write_text("fake") + + class DummyTags: + def __init__(self): + self.values = { + "title": ["Song One"], + "artist": ["Artist One"], + "albumartist": ["Album Artist"], + "album": ["Album One"], + "tracknumber": ["03/12"], + "discnumber": ["2/3"], + } + + def get(self, key, default=None): + return self.values.get(key, default) + + fake_mutagen = types.ModuleType("mutagen") + fake_mutagen.File = lambda path, easy=True: DummyTags() + monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen) + + metadata = read_staging_file_metadata(str(file_path), file_path.name) + + assert metadata == { + "title": "Song One", + "artist": "Artist One", + "albumartist": "Album Artist", + "album": "Album One", + "track_number": 3, + "disc_number": 2, + } + + +def test_read_staging_file_metadata_falls_back_to_filename_track_number(monkeypatch, tmp_path): + file_path = tmp_path / "07 - Song Two.flac" + file_path.write_text("fake") + + fake_mutagen = types.ModuleType("mutagen") + fake_mutagen.File = lambda path, easy=True: None + monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen) + + metadata = read_staging_file_metadata(str(file_path), file_path.name) + + assert metadata["title"] == "07 - Song Two" + assert metadata["track_number"] == 7 + assert metadata["disc_number"] == 1 diff --git a/tests/test_import_filename.py b/tests/test_import_filename.py new file mode 100644 index 00000000..a5b32ff2 --- /dev/null +++ b/tests/test_import_filename.py @@ -0,0 +1,33 @@ +import pytest + +from core.import_filename import parse_filename_metadata + + +@pytest.mark.parametrize( + "filename,expected", + [ + ( + "01 - Artist One - Title One.mp3", + {"artist": "Artist One", "title": "Title One", "album": "", "track_number": 1}, + ), + ( + "Artist Two - Title Two.flac", + {"artist": "Artist Two", "title": "Title Two", "album": "", "track_number": None}, + ), + ( + r"Artist Three\Album Three\03 - Title Three.ogg", + {"artist": "", "title": "Title Three", "album": "Album Three", "track_number": 3}, + ), + ( + "Loose Song.wav", + {"artist": "", "title": "Loose Song", "album": "", "track_number": None}, + ), + ], +) +def test_parse_filename_metadata_handles_common_patterns(filename, expected): + parsed = parse_filename_metadata(filename) + + assert parsed["artist"] == expected["artist"] + assert parsed["title"] == expected["title"] + assert parsed["album"] == expected["album"] + assert parsed["track_number"] == expected["track_number"] diff --git a/tests/test_import_guards.py b/tests/test_import_guards.py new file mode 100644 index 00000000..a7b95372 --- /dev/null +++ b/tests/test_import_guards.py @@ -0,0 +1,47 @@ +from types import SimpleNamespace + +from core import import_guards as guards + + +class _FakeDB: + def __init__(self, quality_profile): + self._quality_profile = quality_profile + + def get_quality_profile(self): + return self._quality_profile + + +def test_check_flac_bit_depth_rejects_strict_mismatch(monkeypatch): + monkeypatch.setattr( + guards, + "MusicDatabase", + lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": False}}}), + ) + monkeypatch.setattr( + guards, + "_get_config_manager", + lambda: SimpleNamespace(get=lambda _key, default=None: False), + ) + + context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}} + + assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) == ( + "FLAC bit depth mismatch: file is 24-bit, preference is 16-bit" + ) + + +def test_check_flac_bit_depth_allows_fallback_when_enabled(monkeypatch): + monkeypatch.setattr( + guards, + "MusicDatabase", + lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": True}}}), + ) + monkeypatch.setattr( + guards, + "_get_config_manager", + lambda: SimpleNamespace(get=lambda _key, default=None: False), + ) + + context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}} + + assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) is None diff --git a/tests/test_import_paths.py b/tests/test_import_paths.py new file mode 100644 index 00000000..f9a39fab --- /dev/null +++ b/tests/test_import_paths.py @@ -0,0 +1,161 @@ +import core.import_paths as import_paths + + +class _Config: + def __init__(self, values): + self._values = values + + def get(self, key, default=None): + return self._values.get(key, default) + + +def test_sanitize_filename_replaces_illegal_characters(): + assert import_paths.sanitize_filename("AC/DC: Song?") == "AC_DC_ Song_" + assert import_paths.sanitize_filename("AUX.txt").startswith("_") + + +def test_build_simple_download_destination_uses_album_folder(monkeypatch, tmp_path): + config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")}) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + destination, album_name, filename = import_paths.build_simple_download_destination( + { + "search_result": { + "filename": "Album Folder/source.flac", + "album": "Album Folder", + } + }, + str(tmp_path / "source.flac"), + ) + + assert destination == tmp_path / "Transfer" / "Album Folder" / "source.flac" + assert album_name == "Album Folder" + assert filename == "source.flac" + assert destination.parent.exists() + + +def test_build_simple_download_destination_falls_back_to_transfer_root(monkeypatch, tmp_path): + config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")}) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + destination, album_name, filename = import_paths.build_simple_download_destination( + { + "search_result": { + "filename": "source.flac", + "album": "Unknown Album", + } + }, + str(tmp_path / "source.flac"), + ) + + assert destination == tmp_path / "Transfer" / "source.flac" + assert album_name == "" + assert filename == "source.flac" + assert destination.parent.exists() + + +def test_get_file_path_from_template_raw_handles_quality_and_disc_placeholders(monkeypatch): + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config({})) + + folder_path, filename = import_paths.get_file_path_from_template_raw( + "$artist/$album/$discnum - $title [$quality]", + { + "artist": "Artist One", + "album": "Album One", + "title": "Song One", + "quality": "FLAC 16bit", + "disc_number": 3, + }, + ) + + assert folder_path == "Artist One/Album One" + assert filename == "3 - Song One [FLAC 16bit]" + + +def test_resolve_album_group_upgrades_standard_to_deluxe(): + import_paths._album_name_cache.clear() + import_paths._album_editions.clear() + + artist_context = {"name": "Cache Artist"} + standard_album = {"album_name": "Cache Album"} + deluxe_album = {"album_name": "Cache Album (Deluxe Edition)"} + + assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album" + assert import_paths.resolve_album_group(artist_context, deluxe_album) == "Cache Album (Deluxe Edition)" + assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album (Deluxe Edition)" + + +def test_build_final_path_for_track_uses_template_and_disc_folder(monkeypatch, tmp_path): + config = _Config( + { + "soulseek.transfer_path": str(tmp_path / "Transfer"), + "file_organization.enabled": True, + "file_organization.templates": { + "album_path": "$albumartist/$albumartist - $album/$track - $title", + "single_path": "$artist/$artist - $title", + }, + "file_organization.collab_artist_mode": "first", + "file_organization.disc_label": "Disc", + } + ) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + calls = [] + + def fake_get_album_tracks_for_source(source, album_id): + calls.append((source, album_id)) + return { + "items": [ + {"disc_number": 1}, + {"disc_number": 2}, + ] + } + + monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", fake_get_album_tracks_for_source) + + context = { + "artist": {"name": "Artist One"}, + "album": { + "name": "Album One", + "id": "album-1", + "release_date": "2026-01-01", + "total_tracks": 12, + "album_type": "album", + "artists": [{"name": "Artist One"}], + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 4, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + "artists": [{"name": "Artist One"}], + }, + "source": "deezer", + "is_album_download": False, + } + + final_path, created = import_paths.build_final_path_for_track( + context, + {"name": "Artist One"}, + { + "is_album": True, + "album_name": "Album One", + "track_number": 4, + "disc_number": 1, + }, + ".flac", + ) + + assert created is True + assert calls == [("deezer", "album-1")] + assert final_path == str( + tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1" / "04 - Song One.flac" + ) + assert (tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1").is_dir() diff --git a/tests/test_import_pipeline.py b/tests/test_import_pipeline.py new file mode 100644 index 00000000..768a8788 --- /dev/null +++ b/tests/test_import_pipeline.py @@ -0,0 +1,128 @@ +import logging +import sys +import types + +import core.import_pipeline as import_pipeline +import core.import_paths as import_paths +import core.import_runtime_state as runtime_state + + +class _Config: + def __init__(self, transfer_path): + self.transfer_path = transfer_path + + def get(self, key, default=None): + if key == "soulseek.transfer_path": + return self.transfer_path + if key in {"post_processing.replaygain_enabled", "lossy_copy.enabled", "lossy_copy.delete_original", "import.replace_lower_quality"}: + return False + return default + + +class _FakeAcoustidVerifier: + def quick_check_available(self): + return False, "disabled" + + +class _ImmediateThread: + def __init__(self, target=None, daemon=None): + self._target = target + + def start(self): + if self._target: + self._target() + + +def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): + transfer_root = tmp_path / "Transfer" + transfer_root.mkdir() + source_path = tmp_path / "source.flac" + source_path.write_bytes(b"audio") + + context_key = "ctx-1" + task_id = "task-1" + batch_id = "batch-1" + context = { + "search_result": { + "is_simple_download": True, + "filename": "Album Folder/source.flac", + "album": "Album Folder", + }, + "track_info": {}, + "original_search_result": {}, + "is_album_download": False, + "task_id": task_id, + "batch_id": batch_id, + } + + mark_calls = [] + completion_calls = [] + scan_calls = [] + activity_calls = [] + + original_matched_context = dict(runtime_state.matched_downloads_context) + original_download_tasks = dict(runtime_state.download_tasks) + original_download_batches = dict(runtime_state.download_batches) + original_processed_ids = set(runtime_state._processed_download_ids) + original_post_process_locks = dict(runtime_state._post_process_locks) + + runtime_state.matched_downloads_context.clear() + runtime_state.download_tasks.clear() + runtime_state.download_batches.clear() + runtime_state._processed_download_ids.clear() + runtime_state._post_process_locks.clear() + + runtime = types.SimpleNamespace( + automation_engine=None, + on_download_completed=lambda batch, task, success: completion_calls.append((batch, task, success)), + web_scan_manager=types.SimpleNamespace(request_scan=lambda reason: scan_calls.append(reason)), + repair_worker=None, + ) + + fake_acoustid = types.ModuleType("core.acoustid_verification") + fake_acoustid.AcoustIDVerification = _FakeAcoustidVerifier + fake_acoustid.VerificationResult = types.SimpleNamespace(FAIL="FAIL") + + monkeypatch.setitem(sys.modules, "core.acoustid_verification", fake_acoustid) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config(str(transfer_root))) + monkeypatch.setattr(import_pipeline, "add_activity_item", lambda *args, **kwargs: activity_calls.append((args, kwargs))) + monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "_mark_task_completed", lambda task, track_info: mark_calls.append((task, track_info))) + monkeypatch.setattr(import_pipeline.threading, "Thread", _ImmediateThread) + + runtime_state.matched_downloads_context[context_key] = context + runtime_state.download_tasks[task_id] = {"track_info": {}, "status": "running"} + + try: + import_pipeline.post_process_matched_download_with_verification( + context_key, + context, + str(source_path), + task_id, + batch_id, + runtime, + ) + + expected_path = transfer_root / "Album Folder" / "source.flac" + assert expected_path.exists() + assert not source_path.exists() + assert context["_simple_download_completed"] is True + assert context["_final_path"] == str(expected_path) + assert mark_calls == [(task_id, {})] + assert completion_calls == [(batch_id, task_id, True)] + assert context_key not in runtime_state.matched_downloads_context + assert scan_calls == ["Simple download completed"] + assert activity_calls + finally: + runtime_state.matched_downloads_context.clear() + runtime_state.matched_downloads_context.update(original_matched_context) + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(original_download_tasks) + runtime_state.download_batches.clear() + runtime_state.download_batches.update(original_download_batches) + runtime_state._processed_download_ids.clear() + runtime_state._processed_download_ids.update(original_processed_ids) + runtime_state._post_process_locks.clear() + runtime_state._post_process_locks.update(original_post_process_locks) diff --git a/tests/test_import_side_effects.py b/tests/test_import_side_effects.py new file mode 100644 index 00000000..510a9ab0 --- /dev/null +++ b/tests/test_import_side_effects.py @@ -0,0 +1,144 @@ +import sqlite3 +from types import SimpleNamespace + +from core import import_side_effects as side_effects + + +class _FakeDB: + def __init__(self, conn): + self._conn = conn + + def _get_connection(self): + return self._conn + + +def _make_soulsync_db(): + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute( + """ + CREATE TABLE artists ( + id TEXT PRIMARY KEY, + name TEXT, + genres TEXT, + thumb_url TEXT, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_artist_id TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id TEXT, + title TEXT, + year INTEGER, + thumb_url TEXT, + genres TEXT, + track_count INTEGER, + duration INTEGER, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_album_id TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT, + artist_id TEXT, + title TEXT, + track_number INTEGER, + duration INTEGER, + file_path TEXT, + bitrate INTEGER, + track_artist TEXT, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_track_id TEXT + ) + """ + ) + return conn + + +def test_record_soulsync_library_entry_writes_artist_album_and_track(tmp_path, monkeypatch): + conn = _make_soulsync_db() + fake_db = _FakeDB(conn) + final_path = tmp_path / "track.flac" + final_path.write_bytes(b"audio") + + monkeypatch.setattr(side_effects, "get_database", lambda: fake_db) + monkeypatch.setattr( + side_effects, + "_get_config_manager", + lambda: SimpleNamespace(get_active_media_server=lambda: "soulsync"), + ) + + import core.genre_filter as genre_filter + + monkeypatch.setattr(genre_filter, "filter_genres", lambda genres, _cfg: [genre.upper() for genre in genres]) + + context = { + "source": "spotify", + "artist": {"id": "sp-artist", "name": "Artist One"}, + "album": { + "id": "sp-album", + "name": "Album One", + "release_date": "2024-02-03", + "total_tracks": 12, + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "id": "sp-track", + "name": "Song One", + "track_number": 7, + "duration_ms": 210000, + "artists": [{"name": "Guest Artist"}], + "_source": "spotify", + }, + "original_search_result": { + "title": "Song One", + "artists": [{"name": "Guest Artist"}], + "_source": "spotify", + }, + "_final_processed_path": str(final_path), + } + + artist_context = {"name": "Artist One", "genres": ["rock", "indie"]} + album_info = {"is_album": True, "album_name": "Album One", "track_number": 7} + + side_effects.record_soulsync_library_entry(context, artist_context, album_info) + + artist_row = conn.execute("SELECT * FROM artists").fetchone() + album_row = conn.execute("SELECT * FROM albums").fetchone() + track_row = conn.execute("SELECT * FROM tracks").fetchone() + + assert artist_row["name"] == "Artist One" + assert artist_row["server_source"] == "soulsync" + assert artist_row["spotify_artist_id"] == "sp-artist" + assert artist_row["genres"] == '["ROCK", "INDIE"]' + + assert album_row["title"] == "Album One" + assert album_row["server_source"] == "soulsync" + assert album_row["spotify_album_id"] == "sp-album" + assert album_row["year"] == 2024 + assert album_row["track_count"] == 12 + assert album_row["duration"] == 210000 + assert album_row["artist_id"] == artist_row["id"] + + assert track_row["title"] == "Song One" + assert track_row["server_source"] == "soulsync" + assert track_row["spotify_track_id"] == "sp-track" + assert track_row["track_number"] == 7 + assert track_row["duration"] == 210000 + assert track_row["track_artist"] == "Guest Artist" + assert track_row["album_id"] == album_row["id"] + assert track_row["file_path"] == str(final_path) diff --git a/tests/test_import_staging.py b/tests/test_import_staging.py new file mode 100644 index 00000000..1cd5e15b --- /dev/null +++ b/tests/test_import_staging.py @@ -0,0 +1,265 @@ +from types import SimpleNamespace + +import core.import_staging as import_staging + + +class FakeClient: + def __init__(self, results=None): + self.results = results or [] + self.calls = [] + + def search_albums(self, query, **kwargs): + self.calls.append((query, kwargs)) + return self.results + + def search_tracks(self, query, **kwargs): + self.calls.append((query, kwargs)) + return self.results + + +def _album_result(album_id, name, artist, release_date="2024-01-01", total_tracks=10, image_url="https://img.example/album.jpg", album_type="album"): + return SimpleNamespace( + id=album_id, + name=name, + artists=[artist], + release_date=release_date, + total_tracks=total_tracks, + image_url=image_url, + album_type=album_type, + ) + + +def test_search_import_albums_prefers_primary_source(monkeypatch): + deezer_client = FakeClient([ + _album_result("deezer-1", "Album One", "Artist One"), + ]) + spotify_client = FakeClient([ + _album_result("spotify-1", "Album One", "Artist One"), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_albums_for_source", + lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit), + ) + + results = import_staging.search_import_albums("Album One", limit=2) + + assert results == [ + { + "id": "deezer-1", + "name": "Album One", + "artist": "Artist One", + "release_date": "2024-01-01", + "total_tracks": 10, + "image_url": "https://img.example/album.jpg", + "album_type": "album", + "source": "deezer", + } + ] + assert deezer_client.calls == [("Album One", {"limit": 2})] + assert spotify_client.calls == [] + + +def test_search_import_albums_falls_back_when_primary_has_no_results(monkeypatch): + deezer_client = FakeClient([]) + spotify_client = FakeClient([ + _album_result("spotify-1", "Album Two", "Artist Two"), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_albums_for_source", + lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit), + ) + + results = import_staging.search_import_albums("Album Two", limit=2) + + assert results == [ + { + "id": "spotify-1", + "name": "Album Two", + "artist": "Artist Two", + "release_date": "2024-01-01", + "total_tracks": 10, + "image_url": "https://img.example/album.jpg", + "album_type": "album", + "source": "spotify", + } + ] + assert deezer_client.calls == [("Album Two", {"limit": 2})] + assert spotify_client.calls == [("Album Two", {"limit": 2, "allow_fallback": False})] + + +def test_search_import_tracks_prefers_primary_source(monkeypatch): + deezer_client = FakeClient([ + SimpleNamespace( + id="deezer-track-1", + name="Song One", + artists=[{"name": "Artist One"}], + album={"id": "deezer-album-1", "name": "Album One"}, + duration_ms=210000, + image_url="https://img.example/track.jpg", + track_number=7, + ), + ]) + spotify_client = FakeClient([ + SimpleNamespace( + id="spotify-track-1", + name="Song One", + artists=["Artist One"], + album="Album One", + duration_ms=210000, + image_url="https://img.example/track.jpg", + track_number=7, + ), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_tracks_for_source", + lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit), + ) + + results = import_staging.search_import_tracks("Song One", limit=2) + + assert results == [ + { + "id": "deezer-track-1", + "name": "Song One", + "artist": "Artist One", + "album": "Album One", + "album_id": "deezer-album-1", + "duration_ms": 210000, + "image_url": "https://img.example/track.jpg", + "track_number": 7, + "source": "deezer", + } + ] + assert deezer_client.calls == [("Song One", {"limit": 2})] + assert spotify_client.calls == [] + + +def test_search_import_tracks_falls_back_when_primary_has_no_results(monkeypatch): + deezer_client = FakeClient([]) + spotify_client = FakeClient([ + SimpleNamespace( + id="spotify-track-1", + name="Song Two", + artists=["Artist Two"], + album="Album Two", + duration_ms=180000, + image_url="", + track_number=3, + ), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_tracks_for_source", + lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit), + ) + + results = import_staging.search_import_tracks("Song Two", limit=2) + + assert results == [ + { + "id": "spotify-track-1", + "name": "Song Two", + "artist": "Artist Two", + "album": "Album Two", + "album_id": "", + "duration_ms": 180000, + "image_url": "", + "track_number": 3, + "source": "spotify", + } + ] + assert deezer_client.calls == [("Song Two", {"limit": 2})] + assert spotify_client.calls == [("Song Two", {"limit": 2, "allow_fallback": False})] + + +def test_build_import_suggestions_background_uses_collected_queries(monkeypatch, tmp_path): + staging_root = tmp_path / "Staging" + staging_root.mkdir() + + cache = import_staging.get_import_suggestions_cache() + original_cache = dict(cache) + cache["suggestions"] = [] + cache["building"] = False + cache["built"] = False + + monkeypatch.setattr(import_staging, "get_staging_path", lambda: str(staging_root)) + monkeypatch.setattr(import_staging, "_collect_import_suggestion_queries", lambda staging_path: ["Album One", "Folder Hint"]) + monkeypatch.setattr( + import_staging, + "search_import_albums", + lambda query, limit=2: [ + { + "id": query.lower().replace(" ", "-"), + "name": query, + "artist": f"{query} Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + } + ], + ) + + try: + import_staging._build_import_suggestions_background() + + assert cache["built"] is True + assert cache["building"] is False + assert cache["suggestions"] == [ + { + "id": "album-one", + "name": "Album One", + "artist": "Album One Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + }, + { + "id": "folder-hint", + "name": "Folder Hint", + "artist": "Folder Hint Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + }, + ] + finally: + cache.clear() + cache.update(original_cache) diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py new file mode 100644 index 00000000..49cbe630 --- /dev/null +++ b/tests/test_metadata_enrichment.py @@ -0,0 +1,333 @@ +import logging +import types + +import pytest + +from core import metadata_enrichment as me + + +class _Config: + def __init__(self, values=None): + self.values = values or {} + + def get(self, key, default=None): + return self.values.get(key, default) + + +class _FakeTag: + def __init__(self, kind, **kwargs): + self.kind = kind + self.kwargs = kwargs + + +class _FakeID3Tags: + def __init__(self): + self.added = [] + + def add(self, frame): + self.added.append(frame) + + def clear(self): + self.added.clear() + + def getall(self, _key): + return [] + + def keys(self): + return [frame.kind for frame in self.added] + + def __len__(self): + return len(self.added) + + +class _FakeAudio: + def __init__(self): + self.tags = _FakeID3Tags() + self.save_calls = [] + self.clear_pictures_calls = 0 + + def clear_pictures(self): + self.clear_pictures_calls += 1 + + def add_tags(self): + self.tags = _FakeID3Tags() + + def save(self, **kwargs): + self.save_calls.append(kwargs) + + +class _FakeResponse: + def __init__(self, payload, content_type="image/jpeg"): + self._payload = payload + self._content_type = content_type + + def read(self): + return self._payload + + def info(self): + return types.SimpleNamespace(get_content_type=lambda: self._content_type) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +def _fake_symbols(audio): + def _tag_factory(kind): + return lambda **kwargs: _FakeTag(kind, **kwargs) + + return types.SimpleNamespace( + File=lambda _path: audio, + APEv2=type("FakeAPEv2", (), {}), + APENoHeaderError=Exception, + FLAC=type("FakeFLAC", (), {}), + Picture=type("FakePicture", (), {"__init__": lambda self: None}), + ID3=_FakeID3Tags, + APIC=_tag_factory("APIC"), + TBPM=_tag_factory("TBPM"), + TCOP=_tag_factory("TCOP"), + TDOR=_tag_factory("TDOR"), + TDRC=_tag_factory("TDRC"), + TCON=_tag_factory("TCON"), + TIT2=_tag_factory("TIT2"), + TALB=_tag_factory("TALB"), + TPE1=_tag_factory("TPE1"), + TPE2=_tag_factory("TPE2"), + TPOS=_tag_factory("TPOS"), + TPUB=_tag_factory("TPUB"), + TRCK=_tag_factory("TRCK"), + TSRC=_tag_factory("TSRC"), + TXXX=_tag_factory("TXXX"), + UFID=_tag_factory("UFID"), + TMED=_tag_factory("TMED"), + MP4=type("FakeMP4", (), {}), + MP4Cover=types.SimpleNamespace(FORMAT_JPEG=1, FORMAT_PNG=2, __call__=lambda *args, **kwargs: ("cover", args, kwargs)), + MP4FreeForm=lambda data: ("freeform", data), + OggVorbis=type("FakeOggVorbis", (), {}), + OggOpus=None, + ) + + +def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(): + class _ItunesClient: + def __init__(self): + self.called = False + + def resolve_primary_artist(self, artist_id): + self.called = True + raise AssertionError("itunes fallback should not run for non-itunes sources") + + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config({"file_organization.collab_artist_mode": "first"}), + itunes_enrichment_worker=types.SimpleNamespace(client=_ItunesClient()), + ) + + context = { + "source": "spotify", + "artist": {"name": "Artist One & Artist Two", "id": "123", "genres": ["rock", "indie"]}, + "album": { + "name": "Album One", + "total_tracks": 12, + "release_date": "2024-01-02", + "images": [{"url": "https://img.example/album.jpg"}], + }, + "track_info": { + "artists": [{"name": "Artist One"}], + "_source": "spotify", + "track_number": 3, + "disc_number": 2, + "total_tracks": 12, + }, + "original_search_result": { + "title": "Song One", + "artists": [{"name": "Artist One"}], + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + "disc_number": 2, + "duration_ms": 180000, + }, + } + + metadata = me.extract_source_metadata( + context, + context["artist"], + {"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 2, "album_image_url": "https://img.example/album.jpg"}, + runtime=runtime, + ) + + assert metadata["source"] == "spotify" + assert metadata["title"] == "Song One" + assert metadata["artist"] == "Artist One" + assert metadata["album_artist"] == "Artist One & Artist Two" + assert metadata["album"] == "Album One" + assert metadata["track_number"] == 3 + assert metadata["total_tracks"] == 12 + assert metadata["disc_number"] == 2 + assert metadata["album_art_url"] == "https://img.example/album.jpg" + assert runtime.itunes_enrichment_worker.client.called is False + + +def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch): + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config(), + mb_worker=None, + deezer_worker=None, + audiodb_worker=None, + tidal_client=None, + qobuz_enrichment_worker=None, + lastfm_worker=None, + genius_worker=None, + itunes_enrichment_worker=None, + get_database=lambda: None, + ) + + audio = _FakeAudio() + symbols = _fake_symbols(audio) + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + + current_metadata = { + "source": "deezer", + "source_track_id": "dz-track", + "source_artist_id": "dz-artist", + "source_album_id": "dz-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + } + me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + + current_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] + assert "DEEZER_TRACK_ID" in current_descs + assert "DEEZER_ARTIST_ID" in current_descs + + audio = _FakeAudio() + symbols = _fake_symbols(audio) + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + + legacy_metadata = { + "source": "", + "spotify_track_id": "sp-track", + "spotify_artist_id": "sp-artist", + "spotify_album_id": "sp-album", + "itunes_track_id": "it-track", + "itunes_artist_id": "it-artist", + "itunes_album_id": "it-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + } + me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + + legacy_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] + assert "SPOTIFY_TRACK_ID" in legacy_descs + assert "SPOTIFY_ARTIST_ID" in legacy_descs + assert "SPOTIFY_ALBUM_ID" in legacy_descs + assert "ITUNES_TRACK_ID" in legacy_descs + assert "ITUNES_ARTIST_ID" in legacy_descs + assert "ITUNES_ALBUM_ID" in legacy_descs + + +def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch): + audio = _FakeAudio() + symbols = _fake_symbols(audio) + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config( + { + "metadata_enhancement.enabled": True, + "metadata_enhancement.embed_album_art": False, + "metadata_enhancement.tags.write_multi_artist": False, + } + ), + mb_worker=None, + deezer_worker=None, + audiodb_worker=None, + tidal_client=None, + qobuz_enrichment_worker=None, + lastfm_worker=None, + genius_worker=None, + itunes_enrichment_worker=None, + get_database=lambda: None, + ) + + strip_calls = [] + verify_calls = [] + + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path, runtime=None: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) + monkeypatch.setattr( + me, + "extract_source_metadata", + lambda context, artist, album_info, runtime=None: { + "source": "deezer", + "source_track_id": "dz-track", + "source_artist_id": "dz-artist", + "source_album_id": "dz-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + "track_number": 3, + "total_tracks": 12, + "disc_number": 2, + "date": "2024", + "genre": "Rock", + "musicbrainz_release_id": "mb-release-1", + }, + ) + monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path, runtime=None: verify_calls.append(file_path) or True) + + album_info = {} + result = me.enhance_file_metadata( + "song.flac", + {"_audio_quality": ""}, + {"name": "Artist One"}, + album_info, + runtime=runtime, + ) + + assert result is True + assert strip_calls == ["song.flac"] + assert verify_calls == ["song.flac"] + assert audio.clear_pictures_calls == 1 + assert len(audio.save_calls) == 2 + assert album_info["musicbrainz_release_id"] == "mb-release-1" + assert any(frame.kind == "TIT2" for frame in audio.tags.added) + assert any(frame.kind == "TPE1" for frame in audio.tags.added) + assert any(frame.kind == "TXXX" and frame.kwargs.get("desc") == "DEEZER_TRACK_ID" for frame in audio.tags.added) + + +def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config( + { + "metadata_enhancement.cover_art_download": True, + "metadata_enhancement.prefer_caa_art": False, + } + ), + ) + + monkeypatch.setattr(me.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes")) + + target_dir = tmp_path / "Album One" + target_dir.mkdir() + + me.download_cover_art( + {}, + str(target_dir), + {"album": {"image_url": "https://img.example/album.jpg"}}, + runtime=runtime, + ) + + cover_path = target_dir / "cover.jpg" + assert cover_path.exists() + assert cover_path.read_bytes() == b"cover-bytes" diff --git a/tests/test_metadata_service_single_import_context.py b/tests/test_metadata_service_single_import_context.py new file mode 100644 index 00000000..ed7ae567 --- /dev/null +++ b/tests/test_metadata_service_single_import_context.py @@ -0,0 +1,193 @@ +from types import SimpleNamespace + +from core import metadata_service + + +class FakeClient: + def __init__(self, search_results=None, details=None, artist_details=None): + self.search_results = search_results or [] + self.details = details or {} + self.artist_details = artist_details or {} + self.calls = [] + + def search_tracks(self, query, limit=1, allow_fallback=True): + self.calls.append(("search_tracks", query, limit, allow_fallback)) + return self.search_results + + def get_track_details(self, track_id): + self.calls.append(("get_track_details", track_id)) + return self.details.get(str(track_id)) + + def get_artist(self, artist_id): + self.calls.append(("get_artist", artist_id)) + return self.artist_details.get(str(artist_id)) + + +def _track_result(track_id="track-1", name="Song One", artist="Artist One"): + return SimpleNamespace( + id=track_id, + name=name, + artists=[artist], + album="Album One", + duration_ms=123000, + track_number=1, + disc_number=1, + image_url="https://img.example/track.jpg", + ) + + +def _track_details(source, track_id="track-1", name="Song One", artist_name="Artist One", artist_id="artist-1"): + return { + "id": track_id, + "name": name, + "track_number": 7, + "disc_number": 1, + "duration_ms": 210000, + "explicit": True, + "uri": f"{source}:track:{track_id}", + "artists": [{"name": artist_name, "id": artist_id}], + "album": { + "id": f"{source}-album-1", + "name": "Album One", + "release_date": "2024-01-01", + "album_type": "album", + "total_tracks": 10, + "images": [{"url": f"https://img.example/{source}-album.jpg"}], + "artists": [{"name": artist_name, "id": artist_id}], + }, + } + + +def test_get_single_track_import_context_uses_primary_source_priority(monkeypatch): + deezer_client = FakeClient( + search_results=[_track_result(track_id="deezer-track-1")], + details={"deezer-track-1": _track_details("deezer", track_id="deezer-track-1", artist_name="Artist One", artist_id="deezer-artist-1")}, + artist_details={"deezer-artist-1": {"id": "deezer-artist-1", "genres": ["electronic"]}}, + ) + spotify_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context("Song One", "Artist One") + + assert result["success"] is True + assert result["source"] == "deezer" + assert result["source_priority"] == ["deezer", "spotify", "itunes"] + assert result["context"]["track_info"]["name"] == "Song One" + assert result["context"]["track_info"]["album_id"] == "deezer-album-1" + assert result["context"]["album"]["image_url"] == "https://img.example/deezer-album.jpg" + assert result["context"]["artist"]["genres"] == ["electronic"] + assert result["context"]["original_search_result"]["clean_title"] == "Song One" + assert deezer_client.calls == [ + ('search_tracks', 'artist:"Artist One" track:"Song One"', 5, True), + ("get_track_details", "deezer-track-1"), + ("get_artist", "deezer-artist-1"), + ] + assert spotify_client.calls == [] + + +def test_get_single_track_import_context_falls_back_to_next_source(monkeypatch): + deezer_client = FakeClient(search_results=[]) + spotify_client = FakeClient( + search_results=[_track_result(track_id="spotify-track-1")], + details={"spotify-track-1": _track_details("spotify", track_id="spotify-track-1", artist_name="Artist Two", artist_id="spotify-artist-1")}, + artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["indie"]}}, + ) + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context("Song Two", "Artist Two") + + assert result["success"] is True + assert result["source"] == "spotify" + assert result["context"]["track_info"]["id"] == "spotify-track-1" + assert result["context"]["album"]["name"] == "Album One" + assert deezer_client.calls == [ + ('search_tracks', 'artist:"Artist Two" track:"Song Two"', 5, True), + ('search_tracks', 'Song Two', 5, True), + ('search_tracks', 'Artist Two', 5, True), + ] + assert spotify_client.calls == [ + ("search_tracks", "Song Two Artist Two", 5, False), + ("get_track_details", "spotify-track-1"), + ("get_artist", "spotify-artist-1"), + ] + + +def test_get_single_track_import_context_uses_explicit_override_first(monkeypatch): + spotify_client = FakeClient( + details={"override-track-1": _track_details("spotify", track_id="override-track-1", artist_name="Override Artist", artist_id="spotify-artist-1")}, + artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["pop"]}}, + ) + deezer_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context( + "Ignored Title", + "Ignored Artist", + override_id="override-track-1", + ) + + assert result["success"] is True + assert result["source"] == "spotify" + assert result["context"]["track_info"]["id"] == "override-track-1" + assert result["context"]["artist"]["genres"] == ["pop"] + assert spotify_client.calls == [ + ("get_track_details", "override-track-1"), + ("get_artist", "spotify-artist-1"), + ] + assert deezer_client.calls == [] + + +def test_get_single_track_import_context_uses_explicit_override_source(monkeypatch): + itunes_client = FakeClient( + details={"override-track-2": _track_details("itunes", track_id="override-track-2", artist_name="Override Artist Two", artist_id="itunes-artist-1")}, + artist_details={"itunes-artist-1": {"id": "itunes-artist-1", "genres": ["singer-songwriter"]}}, + ) + spotify_client = FakeClient() + deezer_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source), + ) + + result = metadata_service.get_single_track_import_context( + "Ignored Title", + "Ignored Artist", + override_id="override-track-2", + override_source="itunes", + ) + + assert result["success"] is True + assert result["source"] == "itunes" + assert result["context"]["track_info"]["id"] == "override-track-2" + assert result["context"]["artist"]["genres"] == ["singer-songwriter"] + assert itunes_client.calls == [ + ("get_track_details", "override-track-2"), + ("get_artist", "itunes-artist-1"), + ] + assert deezer_client.calls == [] + assert spotify_client.calls == [] diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py index dcddd3df..6c08efa0 100644 --- a/tests/test_watchlist_scanner_scan.py +++ b/tests/test_watchlist_scanner_scan.py @@ -2,10 +2,8 @@ import sys import types from datetime import datetime, timedelta - _RECENT_RELEASE_DATE = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") - if "spotipy" not in sys.modules: spotipy = types.ModuleType("spotipy") @@ -1045,6 +1043,10 @@ def test_update_discovery_pool_incremental_uses_source_priority(monkeypatch): assert scanner.database.discovery_pool_calls assert scanner.database.discovery_pool_calls[0][1] == "deezer" + assert scanner.database.discovery_pool_calls[0][0]["deezer_track_id"] == "dz-track-1" + assert scanner.database.discovery_pool_calls[0][0]["deezer_album_id"] == "dz-release-1" + assert scanner.database.discovery_pool_calls[0][0]["deezer_artist_id"] == "dz-artist" + assert scanner.database.discovery_pool_calls[0][0].get("spotify_track_id") is None assert deezer_client.search_calls == [("Incremental Artist", 1, {})] assert deezer_client.album_calls assert spotify_client.search_calls == [] diff --git a/web_server.py b/web_server.py index 8d2a06eb..504ba7b5 100644 --- a/web_server.py +++ b/web_server.py @@ -26,6 +26,9 @@ from flask import Flask, render_template, request, jsonify, redirect, send_file, from flask_socketio import SocketIO, emit, join_room, leave_room from utils.logging_config import get_logger, setup_logging from utils.async_helpers import run_async +from mutagen.flac import FLAC +from mutagen.mp4 import MP4 +from mutagen.oggvorbis import OggVorbis # --- Core Application Imports --- # Import the same core clients and config manager used by the GUI app @@ -94,6 +97,56 @@ from core.database_update_worker import DatabaseUpdateWorker from core.web_scan_manager import WebScanManager from core.lyrics_client import lyrics_client from core.metadata_cache import get_metadata_cache +from core.import_context import ( + build_import_album_info, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_has_clean_metadata, + get_import_has_full_metadata, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + get_library_source_id_columns, + get_source_tag_names, + normalize_import_context, +) +from core.import_album import ( + build_album_import_context, + build_album_import_match_payload, + resolve_album_artist_context, +) +from core.import_album_naming import resolve_album_group as _resolve_album_group +from core.import_filename import extract_track_number_from_filename, parse_filename_metadata +from core.import_staging import ( + get_import_suggestions_cache, + get_primary_source, + get_staging_path, + read_staging_file_metadata, + refresh_import_suggestions_cache, + search_import_albums, + search_import_tracks, + start_import_suggestions_cache, +) +from core.import_paths import build_final_path_for_track as _build_final_path_for_track +from core.metadata_common import get_file_lock +from core.import_runtime_state import ( + activity_feed, + activity_feed_lock, + add_activity_item, + download_batches, + download_tasks, + matched_context_lock, + matched_downloads_context, + mark_task_completed as _core_mark_task_completed, + set_activity_toast_emitter, + tasks_lock, +) +from core import metadata_enrichment +from core.metadata_source import normalize_album_cache_key from database.music_database import get_database, MusicDatabase from services.sync_service import PlaylistSyncService @@ -225,6 +278,7 @@ _socketio_cors_origins = _resolve_socketio_cors_origins(config_manager) socketio = SocketIO(app, async_mode='threading', cors_allowed_origins=_socketio_cors_origins) _log_socketio_startup_status(_socketio_cors_origins, logger) _socketio_rejection_logger = _SocketIORejectionLogger(logger) +set_activity_toast_emitter(socketio.emit) # Plex PIN auth requests stored in memory for polling _plex_pin_requests = {} @@ -1883,7 +1937,7 @@ def _register_automation_handlers(): # --- 4. Sweep empty staging directories --- _update_automation_progress(automation_id, phase='Sweeping import folder...', progress=60) - staging_path = _get_staging_path() + staging_path = get_staging_path() s_removed = 0 if os.path.isdir(staging_path): for dirpath, _dirnames, _filenames in os.walk(staging_path, topdown=False): @@ -2162,346 +2216,6 @@ def _register_automation_handlers(): logger.info("Automation action handlers registered") - -def _emit_track_downloaded(context): - """Emit track_downloaded event for automation engine. Safe to call anywhere.""" - try: - if not automation_engine: - return - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - automation_engine.emit('track_downloaded', { - 'artist': artist_name, - 'title': ti.get('name', ti.get('title', '')), - 'album': ti.get('album', ''), - 'quality': context.get('_audio_quality', 'Unknown'), - }) - except Exception: - pass - - -def _record_library_history_download(context): - """Record a completed download to the library_history table. Non-blocking.""" - try: - # Determine download source - search_result = context.get('original_search_result') or context.get('search_result') or {} - username = search_result.get('username', context.get('_download_username', '')) - _svc_map = {'youtube': 'YouTube', 'tidal': 'Tidal', 'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr'} - download_source = _svc_map.get(username, 'Soulseek') - - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - if not artist_name: - artist_name = ti.get('artist', '') - - album_raw = ti.get('album', '') - album_name = album_raw.get('name', '') if isinstance(album_raw, dict) else str(album_raw or '') - - title = ti.get('name', ti.get('title', '')) - quality = context.get('_audio_quality', '') - file_path = context.get('_final_processed_path', context.get('_final_path', '')) - - # Try to get album art URL - thumb_url = '' - spotify_album = context.get('spotify_album') - if spotify_album and isinstance(spotify_album, dict): - thumb_url = spotify_album.get('image_url', '') - if not thumb_url: - images = spotify_album.get('images', []) - if images: - thumb_url = images[0].get('url', '') - if not thumb_url: - album_info = context.get('album_info', {}) - if isinstance(album_info, dict): - thumb_url = album_info.get('album_image_url', '') - - # Source provenance — what file/track was actually downloaded - source_filename = search_result.get('filename', '') - # Track ID: try search result first, then track_info (Spotify ID used for streaming lookups) - source_track_id = (search_result.get('track_id', '') - or search_result.get('id', '') - or ti.get('id', '')) - - # Source title/artist — what the download source said the track was. - # For Soulseek: parsed from the peer's filename by TrackResult._parse_filename_metadata() - # For Tidal/YouTube/Qobuz: from the streaming API's own metadata - # These live on the candidate's original fields, NOT the spotify_clean_* fields - source_track_title = search_result.get('title', '') or search_result.get('name', '') - source_artist = search_result.get('artist', '') - # For streaming sources, track ID is encoded in filename as "id||display_name" - if source_filename and '||' in source_filename and username in ('tidal', 'youtube', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): - _stream_id = source_filename.split('||')[0] - if _stream_id and not source_track_id: - source_track_id = _stream_id - - # AcoustID verification result - acoustid_result = context.get('_acoustid_result', '') - - db = get_database() - db.add_library_history_entry( - event_type='download', - title=title, - artist_name=artist_name, - album_name=album_name, - quality=quality, - file_path=file_path, - thumb_url=thumb_url, - download_source=download_source, - source_track_id=source_track_id, - source_track_title=source_track_title, - source_filename=source_filename, - acoustid_result=acoustid_result, - source_artist=source_artist - ) - except Exception: - pass # Non-critical, never block download flow - - -def _record_download_provenance(context): - """Record download source provenance for track lineage tracking. Non-blocking.""" - try: - # Extract source info - search_result = context.get('original_search_result') or context.get('search_result') or {} - username = search_result.get('username', context.get('_download_username', '')) - filename = search_result.get('filename', '') - - # Determine source service from username - service_map = {'youtube': 'youtube', 'tidal': 'tidal', 'qobuz': 'qobuz', 'hifi': 'hifi', 'deezer_dl': 'deezer', 'lidarr': 'lidarr'} - source_service = service_map.get(username, 'soulseek') - - # Track metadata - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - if not artist_name: - artist_name = ti.get('artist', '') - - album_raw = ti.get('album', '') - album_name = album_raw.get('name', '') if isinstance(album_raw, dict) else str(album_raw or '') - title = ti.get('name', ti.get('title', '')) - - file_path = context.get('_final_processed_path', context.get('_final_path', '')) - quality = context.get('_audio_quality', '') - size = search_result.get('size', 0) - - # Read audio details from the file for provenance (survives transcoding) - bit_depth = None - sample_rate = None - bitrate = None - try: - if file_path and os.path.isfile(file_path): - from mutagen import File as MutagenFile - audio = MutagenFile(file_path) - if audio and audio.info: - sample_rate = getattr(audio.info, 'sample_rate', None) - bitrate = getattr(audio.info, 'bitrate', None) - bit_depth = getattr(audio.info, 'bits_per_sample', None) - except Exception: - pass - - db = get_database() - db.record_track_download( - file_path=file_path, - source_service=source_service, - source_username=username, - source_filename=filename, - source_size=size or 0, - audio_quality=quality, - track_title=title, - track_artist=artist_name, - track_album=album_name, - bit_depth=bit_depth, - sample_rate=sample_rate, - bitrate=bitrate, - ) - except Exception: - pass # Non-critical, never block download flow - - -def _record_soulsync_library_entry(context, spotify_artist, album_info): - """Write artist/album/track to library DB after successful download/import. - - Only runs when active server is 'soulsync' (standalone mode). Creates - DB records with server_source='soulsync' and pre-populates enrichment - IDs so enrichment workers don't need to re-discover them. - """ - try: - active_server = config_manager.get_active_media_server() - if active_server != 'soulsync': - return - - final_path = context.get('_final_processed_path') - if not final_path: - return - - spotify_album = context.get('spotify_album', {}) or {} - track_info = context.get('track_info', {}) or {} - original_search = context.get('original_search_result', {}) or {} - - artist_name = (spotify_artist or {}).get('name', '') - if not artist_name: - artist_name = original_search.get('spotify_clean_artist', '') or original_search.get('artist', '') - if not artist_name or artist_name in ('Unknown', 'Unknown Artist'): - return - - album_name = '' - if album_info and isinstance(album_info, dict): - album_name = album_info.get('album_name', '') - if not album_name: - album_name = spotify_album.get('name', '') or original_search.get('album', '') - if not album_name: - album_name = track_info.get('name', 'Unknown') - - track_name = original_search.get('spotify_clean_title', '') or track_info.get('name', '') or original_search.get('title', '') - track_number = (track_info.get('track_number') or (album_info.get('track_number') if isinstance(album_info, dict) else None)) or 1 - duration_ms = track_info.get('duration_ms', 0) or 0 - - year = None - release_date = spotify_album.get('release_date', '') - if release_date and len(release_date) >= 4: - try: - year = int(release_date[:4]) - except ValueError: - pass - - image_url = spotify_album.get('image_url', '') - if not image_url: - images = spotify_album.get('images', []) - if images and isinstance(images, list) and len(images) > 0: - img = images[0] - image_url = img.get('url', '') if isinstance(img, dict) else str(img) - - # Enrichment IDs from context — saves enrichment workers from re-discovering - spotify_artist_id = (spotify_artist or {}).get('id', '') - if spotify_artist_id in ('auto_import', 'from_sync_modal', 'explicit_artist', ''): - spotify_artist_id = '' - spotify_album_id = spotify_album.get('id', '') - if spotify_album_id in ('from_sync_modal', 'explicit_album', ''): - spotify_album_id = '' - spotify_track_id = track_info.get('id', '') or original_search.get('id', '') - - genres = (spotify_artist or {}).get('genres', []) - if genres: - from core.genre_filter import filter_genres as _gf2 - genres = _gf2(genres, config_manager) - genres_json = json.dumps(genres) if genres else '' - - bitrate = 0 - try: - from mutagen import File as MutagenFile - audio = MutagenFile(final_path) - if audio and hasattr(audio, 'info') and audio.info and hasattr(audio.info, 'bitrate'): - bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0 - except Exception: - pass - - import hashlib - def _sid(text): - return str(abs(int(hashlib.md5(text.encode('utf-8', errors='replace')).hexdigest(), 16)) % (10 ** 9)) - - artist_id = _sid(artist_name.lower().strip()) - album_id = _sid(f"{artist_name}::{album_name}".lower().strip()) - track_id = _sid(final_path) - total_tracks = spotify_album.get('total_tracks', 0) or 0 - - db = get_database() - with db._get_connection() as conn: - cursor = conn.cursor() - - # ── Artist: find existing soulsync record or create ── - cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,)) - if not cursor.fetchone(): - # Check if soulsync artist exists by name - cursor.execute("SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1", (artist_name,)) - existing_by_name = cursor.fetchone() - if existing_by_name: - artist_id = existing_by_name[0] - else: - # Avoid PK collision with other server sources - cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,)) - if cursor.fetchone(): - # ID taken by another source — append suffix - artist_id = _sid(artist_name.lower().strip() + '::soulsync') - cursor.execute(""" - INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at) - VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (artist_id, artist_name, genres_json, image_url)) - if spotify_artist_id: - try: - cursor.execute("UPDATE artists SET spotify_artist_id = ? WHERE id = ?", (spotify_artist_id, artist_id)) - except Exception: - pass - - # ── Album: find existing soulsync record or create ── - cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,)) - if not cursor.fetchone(): - cursor.execute("SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1", - (album_name, artist_id)) - existing_album_by_name = cursor.fetchone() - if existing_album_by_name: - album_id = existing_album_by_name[0] - else: - # Avoid PK collision - cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,)) - if cursor.fetchone(): - album_id = _sid(f"{artist_name}::{album_name}::soulsync".lower().strip()) - cursor.execute(""" - INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count, - duration, server_source, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms)) - if spotify_album_id: - try: - cursor.execute("UPDATE albums SET spotify_album_id = ? WHERE id = ?", (spotify_album_id, album_id)) - except Exception: - pass - - # ── Track ── - # Determine per-track artist (for compilations/features where track artist != album artist) - track_artist = None - track_artists_list = track_info.get('artists', []) or original_search.get('artists', []) - if track_artists_list: - first_track_artist = track_artists_list[0] - if isinstance(first_track_artist, dict): - ta_name = first_track_artist.get('name', '') - else: - ta_name = str(first_track_artist) - if ta_name and ta_name.lower() != artist_name.lower(): - track_artist = ta_name # Only store when different from album artist - - cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,)) - if not cursor.fetchone(): - cursor.execute(""" - INSERT INTO tracks (id, album_id, artist_id, title, track_number, - duration, file_path, bitrate, track_artist, server_source, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (track_id, album_id, artist_id, track_name, track_number, - duration_ms, final_path, bitrate, track_artist)) - if spotify_track_id: - try: - cursor.execute("UPDATE tracks SET spotify_track_id = ? WHERE id = ?", (spotify_track_id, track_id)) - except Exception: - pass - - conn.commit() - logger.info(f"[SoulSync Library] Added: {artist_name} / {album_name} / {track_name}") - - except Exception as e: - logger.debug(f"[SoulSync Library] Non-critical error: {e}") - - # --- Register Public REST API Blueprint (v1) --- try: from api import create_api_blueprint, limiter @@ -2652,30 +2366,13 @@ def _update_automation_progress(automation_id, **kwargs): pass # --- Global Matched Downloads Context Management --- -# Thread-safe storage for matched download contexts -# Key: slskd download ID, Value: dict containing Spotify artist/album data -matched_downloads_context = {} -matched_context_lock = threading.Lock() +# Shared with core.import_runtime_state so the refactored pipeline and web +# server operate on the same context registry. _orphaned_download_keys = set() # Context keys of downloads abandoned during retry -# --- File-Level Metadata Write Locking --- -# Prevents concurrent threads from writing metadata to the same file simultaneously -_metadata_write_locks = {} # file_path -> threading.Lock() -_metadata_locks_lock = threading.Lock() # Lock for the locks dict - -def _get_file_lock(file_path): - """Get or create a lock for a specific file path to prevent concurrent metadata writes.""" - with _metadata_locks_lock: - if file_path not in _metadata_write_locks: - _metadata_write_locks[file_path] = threading.Lock() - return _metadata_write_locks[file_path] - # --- Download Missing Tracks Modal State Management --- # Thread-safe state tracking for modal download functionality with batch management missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") -download_tasks = {} # task_id -> task state dict -download_batches = {} # batch_id -> {queue, active_count, max_concurrent} -tasks_lock = threading.Lock() batch_locks = {} # batch_id -> Lock() for atomic batch operations def _get_max_concurrent(): @@ -2707,8 +2404,7 @@ def _mark_task_completed(task_id, track_info=None): Assumes task_id exists in download_tasks (should be called within tasks_lock). """ global session_completed_downloads - - download_tasks[task_id]['status'] = 'completed' + _core_mark_task_completed(task_id, track_info) # Increment session counter (matches dashboard.py behavior) with session_stats_lock: @@ -4159,12 +3855,6 @@ _EDITION_BARE_RE = _re.compile( _re.IGNORECASE ) -def _normalize_album_cache_key(album_name): - """Normalize album name for cache key: strip edition suffixes, lowercase, strip whitespace.""" - result = _EDITION_PAREN_RE.sub('', album_name) - result = _EDITION_BARE_RE.sub('', result) - return result.lower().strip() - def _prepare_stream_task(track_data): """ Background streaming task that downloads track to Stream folder and updates global state. @@ -6225,10 +5915,6 @@ def get_debug_info(): return jsonify(info) -# Global activity tracking storage -activity_feed = [] -activity_feed_lock = threading.Lock() - @app.route('/api/activity/feed') def get_activity_feed(): """Get recent activity feed for dashboard""" @@ -6297,37 +5983,6 @@ def get_activity_logs(): except Exception as e: return jsonify({'logs': [f'Error reading activity feed: {str(e)}']}) -def add_activity_item(icon: str, title: str, subtitle: str, time_ago: str = "Now", show_toast: bool = True): - """Add activity item to the feed (replicates dashboard.py functionality)""" - try: - import time - from datetime import datetime, timezone - activity_item = { - 'icon': icon, - 'title': title, - 'subtitle': subtitle, - 'time': datetime.now(timezone.utc).isoformat(), - 'timestamp': time.time(), - 'show_toast': show_toast - } - - with activity_feed_lock: - activity_feed.append(activity_item) - # Keep only last 20 items to prevent memory growth - if len(activity_feed) > 20: - activity_feed.pop(0) - - # Instant toast push via WebSocket (replaces 3-second polling) - if show_toast: - try: - socketio.emit('dashboard:toast', activity_item) - except Exception: - pass - - logger.info(f"Activity: {icon} {title} - {subtitle}") - except Exception as e: - logger.error(f"Error adding activity item: {e}") - # --- Internal API Key Management (browser-only, no auth) --- @app.route('/api/v1/api-keys-internal', methods=['GET']) def list_api_keys_internal(): @@ -13238,7 +12893,7 @@ def write_track_tags(track_id): cover_url = thumb # Use file lock for thread safety - file_lock = _get_file_lock(resolved_path) + file_lock = get_file_lock(resolved_path) with file_lock: result = write_tags_to_file(resolved_path, db_data, embed_cover=embed_cover, cover_url=cover_url) @@ -13396,7 +13051,7 @@ def write_tracks_tags_batch(): if thumb and thumb.startswith('http'): art_data = cover_cache.get(thumb) - file_lock = _get_file_lock(resolved_path) + file_lock = get_file_lock(resolved_path) with file_lock: write_result = write_tags_to_file( resolved_path, db_data, @@ -13523,7 +13178,7 @@ def analyze_track_replaygain(track_id): track_gain_db = _RG_REFERENCE_LUFS - lufs - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: ok = _rg_write_tags(file_path, track_gain_db, peak_dbfs) @@ -13625,7 +13280,7 @@ def analyze_album_replaygain(album_id): continue file_path, track_gain_db, peak_dbfs = entry try: - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: _rg_write_tags(file_path, track_gain_db, peak_dbfs, album_gain_db, album_peak_dbfs) @@ -13711,7 +13366,7 @@ def analyze_tracks_replaygain_batch(): try: lufs, peak_dbfs = _rg_analyze_track(file_path) track_gain_db = _RG_REFERENCE_LUFS - lufs - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: _rg_write_tags(file_path, track_gain_db, peak_dbfs) with _rg_batch_lock: @@ -16562,7 +16217,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() with _mb_release_cache_lock: - _mb_release_cache[(_normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid with _mb_release_detail_cache_lock: _mb_release_detail_cache[_pf_mbid] = _pf_release @@ -16703,7 +16358,7 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() with _mb_release_cache_lock: - _mb_release_cache[(_normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid with _mb_release_detail_cache_lock: _mb_release_detail_cache[_pf_mbid] = _pf_release @@ -16776,9 +16431,6 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): return started_count - - - @app.route('/api/download/matched', methods=['POST']) def start_matched_download(): """ @@ -16903,82 +16555,12 @@ def start_matched_download(): traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 - - - - - def _parse_filename_metadata(filename: str) -> dict: """ A direct port of the metadata parsing logic from the GUI's soulseek_client.py. This is the crucial missing step that cleans filenames BEFORE Spotify matching. """ - import re - import os - - metadata = { - 'artist': None, - 'title': None, - 'album': None, - 'track_number': None - } - - # Get just the filename without extension and path - base_name = os.path.splitext(os.path.basename(filename))[0] - - # --- Logic from soulseek_client.py --- - patterns = [ - # Pattern: 01 - Artist - Title (three-part with track number) - r'^(?P\d{1,2})\s*[-\.]\s*(?P.+?)\s*[-–]\s*(?P.+)$', - # Pattern: 01 - Title (track number + title — must come before Artist - Title - # to prevent "08 - Kilburn Market Dub" matching as artist="08") - r'^(?P<track_number>\d{1,2})\s*[-\.]\s*(?P<title>.+)$', - # Pattern: Artist - Title - r'^(?P<artist>.+?)\s*[-–]\s*(?P<title>.+)$', - ] - - for pattern in patterns: - match = re.match(pattern, base_name) - if match: - match_dict = match.groupdict() - metadata['track_number'] = int(match_dict['track_number']) if match_dict.get('track_number') else None - metadata['artist'] = match_dict.get('artist', '').strip() or None - metadata['title'] = match_dict.get('title', '').strip() or None - break # Stop after first successful match - - # If title is still missing, use the whole base_name - if not metadata['title']: - metadata['title'] = base_name.strip() - - # Fallback for underscore formats like 'Artist_Album_01_Title' - if not metadata['artist'] and '_' in base_name: - parts = base_name.split('_') - if len(parts) >= 3: - # A common pattern is Artist_Album_TrackNum_Title - if parts[-2].isdigit(): - metadata['artist'] = parts[0].strip() - metadata['title'] = parts[-1].strip() - metadata['track_number'] = int(parts[-2]) - metadata['album'] = parts[1].strip() - - # Final cleanup on title if it contains the artist - if metadata['artist'] and metadata['title'] and metadata['artist'].lower() in metadata['title'].lower(): - metadata['title'] = metadata['title'].replace(metadata['artist'], '').lstrip(' -–_').strip() - - - # Try to extract album from the full directory path - if '/' in filename or '\\' in filename: - path_parts = filename.replace('\\', '/').split('/') - if len(path_parts) >= 2: - # The parent directory is often the album - potential_album = path_parts[-2] - # Clean common prefixes like '2024 - ' - cleaned_album = re.sub(r'^\d{4}\s*-\s*', '', potential_album).strip() - metadata['album'] = cleaned_album - - logger.info(f"Parsed Filename '{base_name}': Artist='{metadata['artist']}', Title='{metadata['title']}', Album='{metadata['album']}', Track#='{metadata['track_number']}'") - return metadata - + return parse_filename_metadata(filename) def _read_staging_file_metadata(full_path: str, filename: str) -> dict: """Read metadata from a staging file — tags first, filename parsing as fallback. @@ -16986,57 +16568,7 @@ def _read_staging_file_metadata(full_path: str, filename: str) -> dict: Returns dict with: title, artist, albumartist, album, track_number, disc_number. Only falls back to filename parsing when BOTH title AND artist tags are empty. """ - meta = { - 'title': None, 'artist': None, 'albumartist': None, - 'album': None, 'track_number': None, 'disc_number': None, - } - - # Phase 1: Read embedded tags (most reliable) - try: - from mutagen import File as MutagenFile - tags = MutagenFile(full_path, easy=True) - if tags: - def _first(tag_list): - if isinstance(tag_list, list) and tag_list: - val = str(tag_list[0]).strip() - return val if val else None - return None - - meta['title'] = _first(tags.get('title')) - meta['artist'] = _first(tags.get('artist')) - meta['albumartist'] = _first(tags.get('albumartist')) - meta['album'] = _first(tags.get('album')) - - tn = _first(tags.get('tracknumber')) - if tn: - try: - meta['track_number'] = int(tn.split('/')[0]) - except (ValueError, IndexError): - pass - - dn = _first(tags.get('discnumber')) - if dn: - try: - meta['disc_number'] = int(dn.split('/')[0]) - except (ValueError, IndexError): - pass - except Exception: - pass - - # Phase 2: Only fall back to filename parsing when tags are genuinely empty - if not meta['title'] and not meta['artist']: - parsed = _parse_filename_metadata(filename) - meta['title'] = parsed.get('title') or os.path.splitext(os.path.basename(filename))[0] - meta['artist'] = parsed.get('artist') - if not meta['track_number']: - meta['track_number'] = parsed.get('track_number') - if not meta['album']: - meta['album'] = parsed.get('album') - elif not meta['title']: - # Has artist tag but no title — use filename for title only - meta['title'] = os.path.splitext(os.path.basename(filename))[0] - - return meta + return read_staging_file_metadata(full_path, filename) # =================================================================== @@ -17160,223 +16692,6 @@ def _search_track_in_album_context(original_search: dict, artist: dict) -> dict: return None - - -def _detect_album_info_web(context: dict, artist: dict) -> dict: - """ - Enhanced album detection with GUI parity - multi-priority logic. - (Updated to match GUI downloads.py logic exactly) - """ - try: - # Log available data for debugging (GUI PARITY) - original_search = context.get("original_search_result", {}) - logger.info( - "[Album Detection] start: track=%r clean_spotify_title=%r clean_spotify_album=%r " - "filename_album=%r artist=%r clean_data=%s album_download=%s", - original_search.get('title', 'Unknown'), - original_search.get('spotify_clean_title', 'None'), - original_search.get('spotify_clean_album', 'None'), - original_search.get('album', 'None'), - artist.get('name', 'Unknown'), - context.get('has_clean_spotify_data', False), - context.get('is_album_download', False), - ) - spotify_album_context = context.get("spotify_album") - is_album_download = context.get("is_album_download", False) - artist_name = artist['name'] - - logger.info( - "[Album Detection] track=%r artist=%r has_album_attr=%s album=%r", - original_search.get('title', 'Unknown'), - artist_name, - bool(original_search.get('album')), - original_search.get('album'), - ) - - # --- THIS IS THE CRITICAL FIX --- - # If this is part of a matched album download, we TRUST the context data completely. - # This is the exact logic from downloads.py. - if is_album_download and spotify_album_context: - # We exclusively use the track number and title that were matched - # *before* the download started. We do not try to re-parse the filename. - track_number = original_search.get('track_number', 1) - clean_track_name = original_search.get('title', 'Unknown Track') - - logger.info( - "[Album Detection] using matched context: track_number=%s title=%r album=%r", - track_number, - clean_track_name, - spotify_album_context['name'], - ) - - return { - 'is_album': True, - 'album_name': spotify_album_context['name'], - 'track_number': track_number, - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album_context.get('image_url') - } - - # PRIORITY 1: Try album-aware search using clean Spotify album name (GUI PARITY) - # Prioritize clean Spotify album name over filename-parsed album - clean_album_name = original_search.get('spotify_clean_album') - fallback_album_name = original_search.get('album') - - album_name_to_use = None - album_source = None - - if clean_album_name and clean_album_name.strip() and clean_album_name != "Unknown Album": - album_name_to_use = clean_album_name - album_source = "CLEAN_SPOTIFY" - elif fallback_album_name and fallback_album_name.strip() and fallback_album_name != "Unknown Album": - album_name_to_use = fallback_album_name - album_source = "FILENAME_PARSED" - - if album_name_to_use: - track_title = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown') - logger.info(f"ALBUM-AWARE SEARCH ({album_source}): Looking for '{track_title}' in album '{album_name_to_use}'") - - # Temporarily set the album for the search - original_album = original_search.get('album') - original_search['album'] = album_name_to_use - - try: - album_result = _search_track_in_album_context_web(context, artist) - if album_result: - logger.info(f"PRIORITY 1 SUCCESS: Found track using {album_source} album name - FORCING album classification") - return album_result - else: - logger.error(f"PRIORITY 1 FAILED: Track not found using {album_source} album name") - finally: - # Restore original album value - if original_album is not None: - original_search['album'] = original_album - else: - original_search.pop('album', None) - - # PRIORITY 2: Fallback to individual track search for clean metadata - logger.info("Searching Spotify for individual track info (PRIORITY 2)...") - - # Clean the track title before searching - remove artist prefix - # Prioritize clean Spotify title over filename-parsed title - track_title_to_use = original_search.get('spotify_clean_title') or original_search.get('title', '') - clean_title = _clean_track_title_web(track_title_to_use, artist_name) - logger.info(f"Cleaned title: '{track_title_to_use}' -> '{clean_title}'") - - # Search for the track by artist and cleaned title - query = f"artist:{artist_name} track:{clean_title}" - tracks = spotify_client.search_tracks(query, limit=5) - - # Find the best matching track (prefer album versions over singles) - best_match = None - best_confidence = 0 - - if tracks: - from core.matching_engine import MusicMatchingEngine - matching_engine = MusicMatchingEngine() - for track in tracks: - # Calculate confidence based on artist and title similarity - artist_confidence = matching_engine.similarity_score( - matching_engine.normalize_string(artist_name), - matching_engine.normalize_string(track.artists[0] if track.artists else '') - ) - title_confidence = matching_engine.similarity_score( - matching_engine.normalize_string(clean_title), - matching_engine.normalize_string(track.name) - ) - - combined_confidence = (artist_confidence * 0.6 + title_confidence * 0.4) - - # Small bonus for album tracks so they win ties over singles/EPs - album_type = getattr(track, 'album_type', None) or '' - if album_type == 'album': - combined_confidence += 0.02 - elif album_type == 'ep': - combined_confidence += 0.01 - - if combined_confidence > best_confidence and combined_confidence > 0.75: # Higher threshold to avoid bad matches - best_match = track - best_confidence = combined_confidence - - # If we found a good Spotify match, use it for clean metadata - if best_match and best_confidence > 0.75: - logger.info(f"Found matching Spotify track: '{best_match.name}' - Album: '{best_match.album}' (confidence: {best_confidence:.2f})") - - # Get detailed track information using Spotify's track API - detailed_track = None - if hasattr(best_match, 'id') and best_match.id: - logger.info(f"Getting detailed track info from Spotify API for track ID: {best_match.id}") - detailed_track = spotify_client.get_track_details(best_match.id) - - # Use detailed track data if available - if detailed_track: - logger.info("Got detailed track data from Spotify API") - album_name = _clean_album_title_web(detailed_track['album']['name'], artist_name) - clean_track_name = detailed_track['name'] # Use Spotify's clean track name - album_type = detailed_track['album'].get('album_type', 'album') - total_tracks = detailed_track['album'].get('total_tracks', 1) - spotify_track_number = detailed_track.get('track_number', 1) - - logger.info(f"Spotify album info: '{album_name}' (type: {album_type}, total_tracks: {total_tracks}, track#: {spotify_track_number})") - logger.info(f"Clean track name from Spotify: '{clean_track_name}'") - - # Enhanced album detection using detailed API data (GUI PARITY) - is_album = ( - # Album type is 'album' (not 'single') - album_type == 'album' and - # Album has multiple tracks - total_tracks > 1 and - # Album name different from track name - matching_engine.normalize_string(album_name) != matching_engine.normalize_string(clean_track_name) and - # Album name is not just the artist name - matching_engine.normalize_string(album_name) != matching_engine.normalize_string(artist_name) - ) - - album_image_url = None - if detailed_track['album'].get('images'): - album_image_url = detailed_track['album']['images'][0].get('url') - - logger.info(f"Album classification: {is_album} (type={album_type}, tracks={total_tracks})") - - return { - 'is_album': is_album, - 'album_name': album_name, - 'track_number': spotify_track_number, - 'clean_track_name': clean_track_name, - 'album_image_url': album_image_url, - 'confidence': best_confidence, - 'source': 'spotify_api_detailed' - } - - # Fallback: Use original data with basic cleaning - logger.warning("No good Spotify match found, using original data") - fallback_title = _clean_track_title_web(original_search.get('title', 'Unknown Track'), artist_name) - - # Preserve track_number from context if available (playlist sync tracks have it) - _ctx_track_number = (original_search.get('track_number') - or context.get('track_info', {}).get('track_number') - or 1) - - return { - 'is_album': False, - 'clean_track_name': fallback_title, - 'album_name': fallback_title, - 'track_number': _ctx_track_number, - 'confidence': 0.0, - 'source': 'fallback_original' - } - - except Exception as e: - logger.error(f"Error in _detect_album_info_web: {e}") - clean_title = _clean_track_title_web(context.get("original_search_result", {}).get('title', 'Unknown'), artist.get('name', '')) - _err_tn = (context.get("original_search_result", {}).get('track_number') - or context.get('track_info', {}).get('track_number') - or 1) - return {'is_album': False, 'clean_track_name': clean_title, 'album_name': clean_title, 'track_number': _err_tn} - - - - def _cleanup_empty_directories(download_path, moved_file_path): """Cleans up empty directories after a file move, ignoring hidden files.""" import os @@ -18520,333 +17835,6 @@ def _get_audio_quality_string(file_path): logger.debug(f"Could not determine audio quality for {file_path}: {e}") return '' -def _downsample_hires_flac(final_path, context): - """Downsample a 24-bit hi-res FLAC to 16-bit/44.1kHz CD quality. - - Only runs when downsample_hires is enabled and the file is a 24-bit FLAC. - Replaces the original file in-place (write to temp, verify, swap). - - Returns the (possibly renamed) final_path, or None if no conversion needed. - """ - if not config_manager.get('lossy_copy.downsample_hires', False): - return None - - ext = os.path.splitext(final_path)[1].lower() - if ext != '.flac': - return None - - # Check current bit depth — only downsample if hi-res (>16 bit) - try: - from mutagen.flac import FLAC - audio = FLAC(final_path) - original_bits = audio.info.bits_per_sample - original_rate = audio.info.sample_rate - except Exception as e: - logger.error(f"[Downsample] Could not read FLAC info: {e}") - return None - - if original_bits <= 16 and original_rate <= 44100: - return None # Already CD quality or below - - logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz → 16-bit/44100Hz: {os.path.basename(final_path)}") - - ffmpeg_bin = shutil.which('ffmpeg') - if not ffmpeg_bin: - local = os.path.join(os.path.dirname(__file__), 'tools', 'ffmpeg') - if os.path.isfile(local): - ffmpeg_bin = local - else: - logger.warning("[Downsample] ffmpeg not found — skipping hi-res conversion") - return None - - temp_path = final_path + '.tmp.flac' - try: - result = subprocess.run([ - ffmpeg_bin, '-i', final_path, - '-sample_fmt', 's16', - '-ar', '44100', - '-map_metadata', '0', - '-compression_level', '8', - '-y', temp_path - ], capture_output=True, text=True, timeout=300) - - if result.returncode != 0: - logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}") - if os.path.exists(temp_path): - os.remove(temp_path) - return None - - # Verify the output is a valid 16-bit FLAC - if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0: - logger.warning("[Downsample] Output file missing or empty") - if os.path.exists(temp_path): - os.remove(temp_path) - return None - - verify_audio = FLAC(temp_path) - if verify_audio.info.bits_per_sample != 16: - logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting") - os.remove(temp_path) - return None - - # Atomic swap — replace original with downsampled version - os.replace(temp_path, final_path) - logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}") - - # Update QUALITY tag in the new file - new_quality = 'FLAC 16bit' - try: - updated_audio = FLAC(final_path) - updated_audio['QUALITY'] = new_quality - updated_audio.save() - except Exception as tag_err: - logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}") - - # Update context so downstream (lossy copy, metadata) reflects new quality - old_quality = context.get('_audio_quality', '') - context['_audio_quality'] = new_quality - - # If filename contains old quality string (from $quality template), rename - if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path): - new_basename = os.path.basename(final_path).replace(old_quality, new_quality) - new_path = os.path.join(os.path.dirname(final_path), new_basename) - try: - os.rename(final_path, new_path) - logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} → {new_basename}") - # Rename matching lyrics sidecar file if it exists (.lrc or .txt) - for lyrics_ext in ('.lrc', '.txt'): - old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext - if os.path.isfile(old_lyrics): - new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext - os.rename(old_lyrics, new_lyrics) - return new_path - except Exception as rename_err: - logger.error(f"[Downsample] Could not rename file: {rename_err}") - - return final_path - - except subprocess.TimeoutExpired: - logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}") - if os.path.exists(temp_path): - os.remove(temp_path) - except Exception as e: - logger.error(f"[Downsample] Conversion error: {e}") - if os.path.exists(temp_path): - try: - os.remove(temp_path) - except Exception: - pass - return None - - -def _create_lossy_copy(final_path): - """Convert a FLAC file to a lossy codec at the user's configured bitrate. - - Supported codecs: mp3 (libmp3lame), opus (libopus), aac (aac/libfdk_aac). - Only runs when lossy_copy is enabled and the file is a FLAC. - Places the output alongside the FLAC with the same basename. - - Returns the output path if Blasphemy Mode deleted the original, else None. - """ - if not config_manager.get('lossy_copy.enabled', False): - return None - - ext = os.path.splitext(final_path)[1].lower() - if ext != '.flac': - return None - - codec = config_manager.get('lossy_copy.codec', 'mp3').lower() - bitrate = config_manager.get('lossy_copy.bitrate', '320') - - # Opus max per-channel bitrate is 256kbps — cap to avoid encoding failures - if codec == 'opus' and int(bitrate) > 256: - bitrate = '256' - - # Codec configuration: (ffmpeg_codec, extension, quality_label, extra_args) - # -vn strips video/image streams (embedded cover art) which can cause - # conversion failures when the output muxer can't handle image streams - codec_map = { - 'mp3': ('libmp3lame', '.mp3', f'MP3-{bitrate}', ['-vn', '-id3v2_version', '3']), - 'opus': ('libopus', '.opus', f'OPUS-{bitrate}', ['-vn', '-map', '0:a', '-vbr', 'on']), - 'aac': ('aac', '.m4a', f'AAC-{bitrate}', ['-vn', '-movflags', '+faststart']), - } - - if codec not in codec_map: - logger.info(f"[Lossy Copy] Unknown codec '{codec}' — skipping conversion") - return None - - ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec] - out_path = os.path.splitext(final_path)[0] + out_ext - - # If $quality was used in filename, swap FLAC quality for lossy quality - original_quality = _get_audio_quality_string(final_path) - if original_quality: - out_basename = os.path.basename(out_path) - if original_quality in out_basename: - out_basename = out_basename.replace(original_quality, quality_label) - out_path = os.path.join(os.path.dirname(out_path), out_basename) - - ffmpeg_bin = shutil.which('ffmpeg') - if not ffmpeg_bin: - local = os.path.join(os.path.dirname(__file__), 'tools', 'ffmpeg') - if os.path.isfile(local): - ffmpeg_bin = local - else: - logger.warning(f"[Lossy Copy] ffmpeg not found — skipping {codec.upper()} conversion") - return None - - try: - logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}") - cmd = [ - ffmpeg_bin, '-i', final_path, - '-codec:a', ffmpeg_codec, - '-b:a', f'{bitrate}k', - '-map_metadata', '0', - ] + extra_args + ['-y', out_path] - - result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) - - if result.returncode == 0: - logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}") - - # Fix QUALITY tag — the FLAC's tag was copied verbatim by ffmpeg - try: - from mutagen import File as MutagenFile - audio = MutagenFile(out_path) - if audio is not None: - if codec == 'mp3': - from mutagen.id3 import TXXX - audio.tags.add(TXXX(encoding=3, desc='QUALITY', text=[quality_label])) - elif codec == 'opus': - audio['QUALITY'] = [quality_label] - elif codec == 'aac': - from mutagen.mp4 import MP4FreeForm - audio['----:com.apple.iTunes:QUALITY'] = [MP4FreeForm(quality_label.encode('utf-8'))] - audio.save() - except Exception as tag_err: - logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}") - - # Embed cover art from source FLAC into the lossy copy - # Opus/OGG can't inherit FLAC cover art via ffmpeg -map_metadata alone - if codec in ('opus', 'aac'): - try: - from mutagen import File as MutagenFile - from mutagen.flac import FLAC as MutagenFLAC - source_audio = MutagenFLAC(final_path) - pic = None - if source_audio and source_audio.pictures: - pic = source_audio.pictures[0] - - # Fallback: read cover.jpg from the same directory - if not pic: - cover_path = os.path.join(os.path.dirname(final_path), 'cover.jpg') - if os.path.isfile(cover_path): - try: - from mutagen.flac import Picture - with open(cover_path, 'rb') as f: - img_data = f.read() - pic = Picture() - pic.type = 3 # Cover (front) - pic.mime = 'image/jpeg' - pic.desc = 'Cover' - pic.width = 0 - pic.height = 0 - pic.depth = 0 - pic.colors = 0 - pic.data = img_data - logger.warning("[Lossy Copy] Using cover.jpg as art source (FLAC had no embedded art)") - except Exception: - pass - - if pic: - dest_audio = MutagenFile(out_path) - if dest_audio is not None: - if codec == 'opus': - import base64 - from mutagen.oggopus import OggOpus - if isinstance(dest_audio, OggOpus): - # OGG stores pictures as base64-encoded METADATA_BLOCK_PICTURE - import struct - # Build METADATA_BLOCK_PICTURE block - picture_data = ( - struct.pack('>II', pic.type, len(pic.mime.encode('utf-8'))) - + pic.mime.encode('utf-8') - + struct.pack('>I', len(pic.desc.encode('utf-8'))) - + pic.desc.encode('utf-8') - + struct.pack('>IIII', pic.width, pic.height, pic.depth, pic.colors) - + struct.pack('>I', len(pic.data)) - + pic.data - ) - dest_audio['METADATA_BLOCK_PICTURE'] = [base64.b64encode(picture_data).decode('ascii')] - dest_audio.save() - logger.info("[Lossy Copy] Embedded cover art in Opus file") - elif codec == 'aac': - from mutagen.mp4 import MP4Cover - fmt = MP4Cover.FORMAT_JPEG if 'jpeg' in pic.mime else MP4Cover.FORMAT_PNG - dest_audio['covr'] = [MP4Cover(pic.data, imageformat=fmt)] - dest_audio.save() - logger.info("[Lossy Copy] Embedded cover art in M4A file") - except Exception as art_err: - logger.error(f"[Lossy Copy] Could not embed cover art: {art_err}") - - # Blasphemy Mode: delete original FLAC if enabled and output is verified - if config_manager.get('lossy_copy.delete_original', False): - try: - if os.path.isfile(out_path) and os.path.getsize(out_path) > 0: - from mutagen import File as MutagenFile - test_audio = MutagenFile(out_path) - if test_audio is not None: - # Update provenance record to point to the new transcoded file - try: - db = get_database() - db.update_provenance_file_path(final_path, out_path) - except Exception: - pass - os.remove(final_path) - logger.info(f"[Blasphemy Mode] Deleted original: {os.path.basename(final_path)}") - # Rename lyrics sidecar file to match the output filename - for lyrics_ext in ('.lrc', '.txt'): - src_lyrics = os.path.splitext(final_path)[0] + lyrics_ext - if os.path.isfile(src_lyrics): - dst_lyrics = os.path.splitext(out_path)[0] + lyrics_ext - try: - os.rename(src_lyrics, dst_lyrics) - logger.info(f"[Blasphemy Mode] Renamed {lyrics_ext}: {os.path.basename(src_lyrics)} -> {os.path.basename(dst_lyrics)}") - except Exception as lrc_err: - logger.error(f"[Blasphemy Mode] Could not rename {lyrics_ext}: {lrc_err}") - return out_path - else: - logger.error(f"[Blasphemy Mode] Output failed audio validation, keeping original: {os.path.basename(final_path)}") - else: - logger.warning(f"[Blasphemy Mode] Output missing or empty, keeping original: {os.path.basename(final_path)}") - except Exception as del_err: - logger.error(f"[Blasphemy Mode] Error during original deletion, keeping original: {del_err}") - else: - # ffmpeg always prints its version banner to stderr (~300 chars). - # Strip it so the actual error is visible, and show more than 200 chars. - stderr = result.stderr or '' - # Remove the version/config preamble (ends after the first empty line) - stderr_lines = stderr.split('\n') - error_lines = [] - past_banner = False - for line in stderr_lines: - if past_banner: - error_lines.append(line) - elif line.strip() == '': - past_banner = True - error_msg = '\n'.join(error_lines).strip() if error_lines else stderr[-500:] - logger.error(f"[Lossy Copy] ffmpeg failed (exit code {result.returncode}): {error_msg[:500]}") - # Clean up empty/broken output file - if os.path.isfile(out_path) and os.path.getsize(out_path) == 0: - os.remove(out_path) - logger.warning(f"[Lossy Copy] Removed empty output file: {os.path.basename(out_path)}") - except subprocess.TimeoutExpired: - logger.info(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}") - if os.path.isfile(out_path) and os.path.getsize(out_path) == 0: - os.remove(out_path) - except Exception as e: - logger.error(f"[Lossy Copy] Conversion error: {e}") - return None def _get_album_type_display(raw_type, track_count) -> str: """ @@ -19096,516 +18084,10 @@ from mutagen.apev2 import APEv2, APENoHeaderError import urllib.request def _wipe_source_tags(file_path: str) -> bool: - """Emergency tag wipe — clears ALL tags from a file without writing new ones. - Used when full metadata enhancement is skipped or fails, to prevent original - Soulseek source tags (especially MusicBrainz IDs from the uploader) from - persisting and causing album splits in media servers like Navidrome.""" - try: - _strip_all_non_audio_tags(file_path) - audio = MutagenFile(file_path) - if audio is None: - return False - if hasattr(audio, 'clear_pictures'): - audio.clear_pictures() - if audio.tags is not None: - tag_count = len(audio.tags) - audio.tags.clear() - else: - audio.add_tags() - tag_count = 0 - if isinstance(audio.tags, ID3): - audio.save(v1=0, v2_version=4) - elif isinstance(audio, FLAC): - audio.save(deleteid3=True) - else: - audio.save() - if tag_count > 0: - logger.info(f"[Tag Wipe] Stripped {tag_count} source tags from: {os.path.basename(file_path)}") - return True - except Exception as e: - logger.error(f"[Tag Wipe] Failed (non-fatal): {e}") - return False - - -def _strip_all_non_audio_tags(file_path: str) -> dict: - """ - Strip ALL non-audio tag containers from a file before metadata rewriting. - MP3 files from Soulseek commonly carry APEv2 tags (foobar2000 users) - with stale metadata that Mutagen's ID3 handler cannot see or clear. - Must run BEFORE MutagenFile() opens the file. - """ - summary = {'apev2_stripped': False, 'apev2_tag_count': 0} - ext = os.path.splitext(file_path)[1].lower() - if ext != '.mp3': - return summary - try: - apev2_tags = APEv2(file_path) - tag_count = len(apev2_tags) - tag_keys = list(apev2_tags.keys()) - apev2_tags.delete(file_path) - summary['apev2_stripped'] = True - summary['apev2_tag_count'] = tag_count - logger.info(f"Stripped {tag_count} APEv2 tags: {', '.join(tag_keys[:10])}") - except APENoHeaderError: - pass # No APEv2 tags — common case - except Exception as e: - logger.error(f"Could not strip APEv2 tags (non-fatal): {e}") - return summary - -def _verify_metadata_written(file_path: str) -> bool: - """Re-open file and verify core metadata fields are present.""" - try: - check = MutagenFile(file_path) - if check is None or check.tags is None: - logger.info(f"[VERIFY] Tags are None after save: {file_path}") - return False - title_found = False - artist_found = False - if isinstance(check.tags, ID3): - title_found = bool(check.tags.getall('TIT2')) - artist_found = bool(check.tags.getall('TPE1')) - # Confirm APEv2 is gone - try: - APEv2(file_path) - logger.info("[VERIFY] APEv2 tags still present after processing!") - return False - except APENoHeaderError: - pass - elif isinstance(check, (FLAC, OggVorbis)) or _is_ogg_opus(check): - title_found = bool(check.get('title')) - artist_found = bool(check.get('artist')) - elif isinstance(check, MP4): - title_found = bool(check.get('\xa9nam')) - artist_found = bool(check.get('\xa9ART')) - if not title_found or not artist_found: - logger.warning(f"[VERIFY] Missing metadata - title:{title_found} artist:{artist_found}") - return False - logger.info("[VERIFY] Metadata verified OK") - return True - except Exception as e: - logger.error(f"[VERIFY] Verification error (non-fatal): {e}") - return False - -def _is_ogg_opus(audio_file): - """Check if a Mutagen file object is OggOpus (uses VorbisComment tags like FLAC/OGG).""" - return type(audio_file).__name__ == 'OggOpus' + return metadata_enrichment.wipe_source_tags(file_path) def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - """ - Core function to enhance audio file metadata using Spotify data. - Thread-safe with per-file locking to prevent concurrent metadata writes. - - Opens the file once in non-easy mode, clears all tags in memory, writes - new tags using format-specific frames/keys, embeds album art and source - IDs, then saves once. This avoids the old clear→save→reopen pattern - which stripped the ID3v2 header from MP3 files, leaving them tagless. - """ - if not config_manager.get('metadata_enhancement.enabled', True): - logger.warning("Metadata enhancement disabled in config.") - return True - - # Normalize None album_info to empty dict to prevent AttributeError on .get() calls - if album_info is None: - album_info = {} - - # Acquire per-file lock to prevent concurrent metadata writes to the same file - file_lock = _get_file_lock(file_path) - with file_lock: - logger.info(f"Enhancing metadata for: {os.path.basename(file_path)}") - try: - # Strip APEv2 tags from MP3 (invisible to ID3 handler) - strip_summary = _strip_all_non_audio_tags(file_path) - - audio_file = MutagenFile(file_path) - if audio_file is None: - logger.error(f"Could not load audio file with Mutagen: {file_path}") - return False - - # ── Wipe ALL existing tags and save immediately ── - # Files from Soulseek carry random metadata (wrong comments, - # encoder info, ReplayGain, old album art, random TXXX frames). - # Save the cleared state FIRST so that if anything below throws, - # the file at least has clean (empty) tags instead of junk that - # causes album fragmentation in media servers. - if hasattr(audio_file, 'clear_pictures'): - audio_file.clear_pictures() - - if audio_file.tags is not None: - if len(audio_file.tags) > 0: - tag_keys = list(audio_file.tags.keys())[:15] - logger.info(f"Clearing {len(audio_file.tags)} existing tags: " - f"{', '.join(str(k) for k in tag_keys)}") - audio_file.tags.clear() - else: - audio_file.add_tags() - - # Persist the wipe — guarantees junk tags are gone even if later steps fail - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - - metadata = _extract_spotify_metadata(context, artist, album_info) - if not metadata: - logger.error("Could not extract Spotify metadata, saving with cleared tags.") - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - return True - - # ── Write standard tags using format-specific API ── - track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}" - - _write_multi = config_manager.get('metadata_enhancement.tags.write_multi_artist', False) - _artists_list = metadata.get('_artists_list', []) - - if isinstance(audio_file.tags, ID3): - # MP3: write ID3 frames directly - if metadata.get('title'): - audio_file.tags.add(TIT2(encoding=3, text=[metadata['title']])) - if metadata.get('artist'): - audio_file.tags.add(TPE1(encoding=3, text=[metadata['artist']])) - # Multi-value: write each artist as separate TPE1 text value - if _write_multi and len(_artists_list) > 1: - audio_file.tags.add(TPE1(encoding=3, text=_artists_list)) - if metadata.get('album_artist'): - audio_file.tags.add(TPE2(encoding=3, text=[metadata['album_artist']])) - if metadata.get('album'): - audio_file.tags.add(TALB(encoding=3, text=[metadata['album']])) - if metadata.get('date'): - audio_file.tags.add(TDRC(encoding=3, text=[metadata['date']])) - if metadata.get('genre'): - audio_file.tags.add(TCON(encoding=3, text=[metadata['genre']])) - audio_file.tags.add(TRCK(encoding=3, text=[track_num_str])) - if metadata.get('disc_number'): - audio_file.tags.add(TPOS(encoding=3, text=[str(metadata['disc_number'])])) - - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - # FLAC / OGG Vorbis / OGG Opus: dict-style VorbisComment tags - if metadata.get('title'): - audio_file['title'] = [metadata['title']] - if metadata.get('artist'): - audio_file['artist'] = [metadata['artist']] - # Multi-value: write ARTISTS tag with individual values - if _write_multi and len(_artists_list) > 1: - audio_file['artists'] = _artists_list - if metadata.get('album_artist'): - audio_file['albumartist'] = [metadata['album_artist']] - if metadata.get('album'): - audio_file['album'] = [metadata['album']] - if metadata.get('date'): - audio_file['date'] = [metadata['date']] - if metadata.get('genre'): - audio_file['genre'] = [metadata['genre']] - audio_file['tracknumber'] = [track_num_str] - if metadata.get('disc_number'): - audio_file['discnumber'] = [str(metadata['disc_number'])] - - elif isinstance(audio_file, MP4): - # MP4 / M4A: Apple-style tag keys - if metadata.get('title'): - audio_file['\xa9nam'] = [metadata['title']] - if metadata.get('artist'): - # Multi-value: write each artist as separate list entry - if _write_multi and len(_artists_list) > 1: - audio_file['\xa9ART'] = _artists_list - else: - audio_file['\xa9ART'] = [metadata['artist']] - if metadata.get('album_artist'): - audio_file['aART'] = [metadata['album_artist']] - if metadata.get('album'): - audio_file['\xa9alb'] = [metadata['album']] - if metadata.get('date'): - audio_file['\xa9day'] = [metadata['date']] - if metadata.get('genre'): - audio_file['\xa9gen'] = [metadata['genre']] - track_num = metadata.get('track_number', 1) - total_tracks = metadata.get('total_tracks', 1) - audio_file['trkn'] = [(track_num, total_tracks)] - if metadata.get('disc_number'): - audio_file['disk'] = [(metadata['disc_number'], 0)] - - # ── Embed source IDs (Spotify, MusicBrainz, etc.) on the same object ── - # Runs before album art so MusicBrainz release ID is available for - # Cover Art Archive high-resolution lookup. - _embed_source_ids(audio_file, metadata, context) - - # Propagate MusicBrainz release ID to album_info so _download_cover_art - # can use it for Cover Art Archive high-res cover.jpg - if album_info is not None and metadata.get('musicbrainz_release_id'): - album_info['musicbrainz_release_id'] = metadata['musicbrainz_release_id'] - - # ── Embed album art on the same object ── - if config_manager.get('metadata_enhancement.embed_album_art', True): - _embed_album_art_metadata(audio_file, metadata) - - # ── Embed audio quality tag ── - quality = context.get('_audio_quality', '') - if quality and config_manager.get('metadata_enhancement.tags.quality_tag', True) is not False: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='QUALITY', text=[quality])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['quality'] = [quality] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:QUALITY'] = [MP4FreeForm(quality.encode('utf-8'))] - - # ── Single save for everything ── - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - - # Verify metadata was written - verified = _verify_metadata_written(file_path) - if verified: - logger.info("Metadata enhanced successfully.") - else: - logger.info("Metadata saved but verification found issues (see above).") - return True - except Exception as e: - import traceback - logger.error(f"Error enhancing metadata for {file_path}: {e}") - logger.error(f"[Metadata Debug] Exception type: {type(e).__name__}") - logger.info(f"[Metadata Debug] File exists: {os.path.exists(file_path)}") - logger.warning(f"[Metadata Debug] Artist: {artist.get('name', 'MISSING') if artist else 'None'}") - logger.warning(f"[Metadata Debug] Album info: {album_info.get('album_name', 'MISSING') if album_info else 'None'}") - logger.error(f"[Metadata Debug] Traceback:\n{traceback.format_exc()}") - return False - -def _generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - """ - Generate LRC lyrics file using LRClib API. - Elegant addition to post-processing - extracts metadata from existing context. - """ - if not config_manager.get('metadata_enhancement.lrclib_enabled', True): - return False - try: - # Extract track information from existing context (same as metadata enhancement) - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album") - - # Get track metadata - track_name = (original_search.get('spotify_clean_title') or - original_search.get('title', 'Unknown Track')) - - # Handle artist parameter (can be dict or object) - if isinstance(artist, dict): - artist_name = artist.get('name', 'Unknown Artist') - elif hasattr(artist, 'name'): - artist_name = artist.name - else: - artist_name = str(artist) if artist else 'Unknown Artist' - album_name = None - duration_seconds = None - - # Get album name if available - if album_info.get('is_album'): - album_name = (original_search.get('spotify_clean_album') or - album_info.get('album_name') or - (spotify_album.get('name') if spotify_album else None)) - - # Get duration from original search context - if original_search.get('duration_ms'): - duration_seconds = int(original_search['duration_ms'] / 1000) - - # Generate LRC file using lyrics client - success = lyrics_client.create_lrc_file( - audio_file_path=file_path, - track_name=track_name, - artist_name=artist_name, - album_name=album_name, - duration_seconds=duration_seconds - ) - - if success: - logger.info(f"LRC file generated for: {track_name}") - else: - logger.warning(f"No lyrics found for: {track_name}") - - return success - - except Exception as e: - logger.error(f"Error generating LRC file for {file_path}: {e}") - return False - -def _extract_spotify_metadata(context: dict, artist: dict, album_info: dict) -> dict: - """Extracts a comprehensive metadata dictionary from the provided context.""" - metadata = {} - if album_info is None: - album_info = {} - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album") - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - metadata['title'] = original_search['spotify_clean_title'] - logger.info(f"Metadata: Using Spotify clean title: '{metadata['title']}'") - # Priority 2: Album info clean name - elif album_info.get('clean_track_name'): - metadata['title'] = album_info['clean_track_name'] - logger.info(f"Metadata: Using album info clean name: '{metadata['title']}'") - # Priority 3: Original title as fallback - else: - metadata['title'] = original_search.get('title', '') - logger.warning(f"Metadata: Using original title as fallback: '{metadata['title']}'") - # Handle multiple artists from Spotify data - original_search = context.get("original_search_result", {}) - if 'artists' in original_search and isinstance(original_search['artists'], list) and len(original_search['artists']) > 0: - all_artists = [] - for a in original_search['artists']: - if isinstance(a, dict) and 'name' in a: - all_artists.append(a['name']) - elif isinstance(a, str): - all_artists.append(a) - else: - all_artists.append(str(a)) - metadata['artist'] = ', '.join(all_artists) - logger.info(f"Metadata: Using all artists: '{metadata['artist']}'") - else: - # Fallback to single artist - metadata['artist'] = artist.get('name', '') - logger.info(f"Metadata: Using primary artist: '{metadata['artist']}'") - - # Resolve album_artist for consistent tagging across all tracks in an album. - # Priority: 1) explicit batch artist context (same artist for whole album) - # 2) album-level artists from spotify_album - # 3) context-level artist (spotify_artist parameter) - # 4) collab mode first-artist resolution (per-track, last resort) - # Using album-level artist prevents media server album splits when an artist - # changed names (Kanye West → Ye) and Spotify returns different per-track artists. - _raw_album_artist = artist.get('name', '') - _track_info_ctx = context.get('track_info', {}) or {} - _explicit_aa = _track_info_ctx.get('_explicit_artist_context') if isinstance(_track_info_ctx, dict) else None - - # Build album-level artists list for collab mode resolution. - # Using album-level artists (instead of per-track) ensures collab mode produces - # the SAME album_artist tag for every track, preventing media server album splits. - _album_artists_for_collab = None - if isinstance(_explicit_aa, dict) and _explicit_aa.get('name'): - _raw_album_artist = _explicit_aa['name'] - _album_artists_for_collab = [_explicit_aa] - elif isinstance(_explicit_aa, str) and _explicit_aa: - _raw_album_artist = _explicit_aa - _album_artists_for_collab = [{'name': _explicit_aa}] - elif spotify_album and isinstance(spotify_album, dict): - _sa_aa = spotify_album.get('artists', []) - if _sa_aa: - _first_aa = _sa_aa[0] - if isinstance(_first_aa, dict) and _first_aa.get('name'): - _raw_album_artist = _first_aa['name'] - elif isinstance(_first_aa, str) and _first_aa: - _raw_album_artist = _first_aa - _album_artists_for_collab = _sa_aa - - collab_mode = config_manager.get('file_organization.collab_artist_mode', 'first') - if collab_mode == 'first' and _raw_album_artist: - original_search = context.get("original_search_result", {}) - # Prefer album-level artists for collab resolution (consistent per album) - _ctx_artists = _album_artists_for_collab or original_search.get('artists') or _track_info_ctx.get('artists') or [] - if len(_ctx_artists) > 1: - # Multiple artist objects (Spotify) — use first - first = _ctx_artists[0] - _raw_album_artist = first.get('name', first) if isinstance(first, dict) else str(first) - elif len(_ctx_artists) == 1 and (',' in _raw_album_artist or ' & ' in _raw_album_artist): - # Single combined string (iTunes) — resolve via artist ID - _aid = str(artist.get('id', '')) - _src = original_search.get('_source') or _track_info_ctx.get('_source', '') - if _aid.isdigit() and _src != 'deezer': - try: - resolved = _get_itunes_client().resolve_primary_artist(_aid) - if resolved and resolved != _raw_album_artist: - _raw_album_artist = resolved - except Exception: - pass - metadata['album_artist'] = _raw_album_artist # Crucial for library organization - - if album_info.get('is_album'): - metadata['album'] = album_info.get('album_name', 'Unknown Album') - track_num = album_info.get('track_number', 1) - metadata['track_number'] = track_num - metadata['total_tracks'] = spotify_album.get('total_tracks', 1) if spotify_album else 1 - logger.info(f"[METADATA] Album track - track_number: {track_num}, album: {metadata['album']}") - else: - # SAFEGUARD: If we have spotify_album context, never use track title as album name - # This prevents album tracks from being tagged as singles due to classification errors - if spotify_album and spotify_album.get('name'): - logger.info("[SAFEGUARD] Using spotify_album name instead of track title for album metadata") - metadata['album'] = spotify_album['name'] - # Use corrected track_number from album_info (which should be updated by post-processing) - corrected_track_number = album_info.get('track_number', 1) if album_info else 1 - metadata['track_number'] = corrected_track_number - metadata['total_tracks'] = spotify_album.get('total_tracks', 1) - logger.info(f"[SAFEGUARD] Using track_number: {corrected_track_number}") - else: - metadata['album'] = metadata['title'] # For true singles, album is the title - metadata['track_number'] = 1 - metadata['total_tracks'] = 1 - - # Always write disc_number to overwrite any stale tags from the soulseek source. - # Without this, original disc tags persist and can cause media servers (Plex) to - # split a single album into standard/deluxe based on differing disc numbers. - # Priority: original_search context (from API) > album_info > default to 1 - disc_num = original_search.get('disc_number') - if disc_num is None and album_info: - disc_num = album_info.get('disc_number') - if disc_num is None: - disc_num = 1 - metadata['disc_number'] = disc_num - - if spotify_album and spotify_album.get('release_date'): - metadata['date'] = spotify_album['release_date'][:4] - - if artist.get('genres'): - from core.genre_filter import filter_genres - _genre_list = filter_genres(list(artist['genres'][:2]), config_manager) - if _genre_list: - metadata['genre'] = ', '.join(_genre_list) - - metadata['album_art_url'] = album_info.get('album_image_url') if album_info else None - - # Playlist mode fallback: album_info is None, try to get art from spotify_album context - if not metadata['album_art_url']: - _spa = context.get('spotify_album', {}) - if _spa: - _spa_img = _spa.get('image_url') - if not _spa_img and _spa.get('images'): - _spa_img = _spa['images'][0].get('url') if isinstance(_spa['images'][0], dict) else None - metadata['album_art_url'] = _spa_img - - # Extract source IDs (Spotify or iTunes) for tag embedding - track_info = context.get("track_info", {}) - if track_info and track_info.get('id'): - # Spotify track IDs are alphanumeric strings; iTunes IDs are numeric - # Beatport IDs (beatport_*) are neither — skip them for external ID tagging - track_id = str(track_info['id']) - if track_id.isdigit(): - metadata['itunes_track_id'] = track_id - elif not track_id.startswith('beatport_'): - metadata['spotify_track_id'] = track_id - if artist.get('id'): - artist_id = str(artist['id']) - if artist_id.isdigit(): - metadata['itunes_artist_id'] = artist_id - elif not artist_id.startswith('beatport_'): - metadata['spotify_artist_id'] = artist_id - if spotify_album and spotify_album.get('id'): - album_id = str(spotify_album['id']) - if album_id.isdigit(): - metadata['itunes_album_id'] = album_id - elif not album_id.startswith('beatport_'): - metadata['spotify_album_id'] = album_id - - # Summary log for debugging metadata issues (e.g. wrong album_artist / track_number) - logger.info(f"[Metadata Summary] title='{metadata.get('title')}' | artist='{metadata.get('artist')}' | album_artist='{metadata.get('album_artist')}' | album='{metadata.get('album')}' | track={metadata.get('track_number')}/{metadata.get('total_tracks')} | disc={metadata.get('disc_number')}") - - return metadata + return metadata_enrichment.enhance_file_metadata(file_path, context, artist, album_info) def _get_image_dimensions(data: bytes): """Extract width/height from JPEG or PNG image data without PIL.""" @@ -19633,870 +18115,12 @@ def _get_image_dimensions(data: bytes): return None, None -def _embed_album_art_metadata(audio_file, metadata: dict): - """Downloads and embeds album art — tries Cover Art Archive (full resolution) - first if MusicBrainz release ID is available, falls back to Spotify/iTunes URL.""" - try: - image_data = None - mime_type = None - - # Try Cover Art Archive first (often 1200x1200+, original quality) — opt-in - release_mbid = metadata.get('musicbrainz_release_id') - if release_mbid and config_manager.get('metadata_enhancement.prefer_caa_art', False): - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={'Accept': 'image/*'}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or 'image/jpeg' - if image_data and len(image_data) > 1000: - logger.info(f"Cover art from Cover Art Archive ({len(image_data) // 1024}KB)") - else: - image_data = None # Too small, likely an error page - except Exception: - image_data = None # Fall through to Spotify/iTunes URL - - # Fallback to Spotify/iTunes/Deezer URL (typically 640x640) - if not image_data: - art_url = metadata.get('album_art_url') - if not art_url: - logger.warning("No album art URL available for embedding.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() - - if not image_data: - logger.error("Failed to download album art data.") - return - - # MP3 (ID3) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(APIC(encoding=3, mime=mime_type, type=3, desc='Cover', data=image_data)) - # FLAC - elif isinstance(audio_file, FLAC): - picture = Picture() - picture.data = image_data - picture.type = 3 - picture.mime = mime_type - # Detect actual dimensions from image data - _img_w, _img_h = _get_image_dimensions(image_data) - picture.width = _img_w or 640 - picture.height = _img_h or 640 - picture.depth = 24 - audio_file.add_picture(picture) - # MP4/M4A - elif isinstance(audio_file, MP4): - fmt = MP4Cover.FORMAT_JPEG if 'jpeg' in mime_type else MP4Cover.FORMAT_PNG - audio_file['covr'] = [MP4Cover(image_data, imageformat=fmt)] - - logger.info("Album art successfully embedded.") - except Exception as e: - logger.error(f"Error embedding album art: {e}") - -def _embed_source_ids(audio_file, metadata: dict, context: dict = None): - """ - Lookup MusicBrainz, Deezer, AudioDB, Tidal, Qobuz, Last.fm, and Genius - metadata, then embed them along with Spotify/iTunes source IDs as custom - tags into the audio file. - Tags written: source IDs, BPM (Deezer), mood/style (AudioDB), ISRC - (MB→Deezer→Tidal→Qobuz fallback), copyright (Tidal→Qobuz), - label (Qobuz), URLs (Last.fm/Genius), - and merged genres (Spotify+MB+AudioDB+Last.fm). - One file write, one shot. Concurrent calls are safe — each service has - its own global rate limiter. - Operates on a non-easy-mode MutagenFile object (caller must save). - """ - try: - # ── Per-tag config: maps internal tag name → config path ── - # Each tag can be individually toggled via {service}.tags.{tag_name} - _TAG_CONFIG = { - # Spotify (from metadata, no API call) - 'SPOTIFY_TRACK_ID': 'spotify.tags.track_id', - 'SPOTIFY_ARTIST_ID': 'spotify.tags.artist_id', - 'SPOTIFY_ALBUM_ID': 'spotify.tags.album_id', - # iTunes (from metadata, no API call) - 'ITUNES_TRACK_ID': 'itunes.tags.track_id', - 'ITUNES_ARTIST_ID': 'itunes.tags.artist_id', - 'ITUNES_ALBUM_ID': 'itunes.tags.album_id', - # MusicBrainz IDs - 'MUSICBRAINZ_RECORDING_ID': 'musicbrainz.tags.recording_id', - 'MUSICBRAINZ_ARTIST_ID': 'musicbrainz.tags.artist_id', - 'MUSICBRAINZ_RELEASE_ID': 'musicbrainz.tags.release_id', - 'MUSICBRAINZ_RELEASEGROUPID': 'musicbrainz.tags.release_group_id', - 'MUSICBRAINZ_ALBUMARTISTID': 'musicbrainz.tags.album_artist_id', - 'MUSICBRAINZ_RELEASETRACKID': 'musicbrainz.tags.release_track_id', - # MusicBrainz Release Info - 'RELEASETYPE': 'musicbrainz.tags.release_type', - 'ORIGINALDATE': 'musicbrainz.tags.original_date', - 'RELEASESTATUS': 'musicbrainz.tags.release_status', - 'RELEASECOUNTRY': 'musicbrainz.tags.release_country', - 'BARCODE': 'musicbrainz.tags.barcode', - 'MEDIA': 'musicbrainz.tags.media', - 'TOTALDISCS': 'musicbrainz.tags.total_discs', - 'CATALOGNUMBER': 'musicbrainz.tags.catalog_number', - 'SCRIPT': 'musicbrainz.tags.script', - 'ASIN': 'musicbrainz.tags.asin', - # Deezer - 'DEEZER_TRACK_ID': 'deezer.tags.track_id', - 'DEEZER_ARTIST_ID': 'deezer.tags.artist_id', - # AudioDB - 'AUDIODB_TRACK_ID': 'audiodb.tags.track_id', - # Tidal - 'TIDAL_TRACK_ID': 'tidal.tags.track_id', - 'TIDAL_ARTIST_ID': 'tidal.tags.artist_id', - # Qobuz - 'QOBUZ_TRACK_ID': 'qobuz.tags.track_id', - 'QOBUZ_ARTIST_ID': 'qobuz.tags.artist_id', - # Genius - 'GENIUS_TRACK_ID': 'genius.tags.track_id', - } - - def _tag_enabled(config_path): - """Check if an individual tag is enabled (defaults to True).""" - return config_manager.get(config_path, True) is not False - - # ── Helper: normalize + compare names (same logic as enrichment workers) ── - from difflib import SequenceMatcher - def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: - if not a or not b: - return False - norm = lambda s: re.sub(r'[^a-z0-9 ]', '', re.sub(r'\(.*?\)', '', s).lower()).strip() - return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold - - # ── 1. Collect Spotify / iTunes IDs already in metadata ── - id_tags = {} - if config_manager.get('spotify.embed_tags', True) is not False: - if metadata.get('spotify_track_id'): - id_tags['SPOTIFY_TRACK_ID'] = metadata['spotify_track_id'] - if metadata.get('spotify_artist_id'): - id_tags['SPOTIFY_ARTIST_ID'] = metadata['spotify_artist_id'] - if metadata.get('spotify_album_id'): - id_tags['SPOTIFY_ALBUM_ID'] = metadata['spotify_album_id'] - if config_manager.get('itunes.embed_tags', True) is not False: - if metadata.get('itunes_track_id'): - id_tags['ITUNES_TRACK_ID'] = metadata['itunes_track_id'] - if metadata.get('itunes_artist_id'): - id_tags['ITUNES_ARTIST_ID'] = metadata['itunes_artist_id'] - if metadata.get('itunes_album_id'): - id_tags['ITUNES_ALBUM_ID'] = metadata['itunes_album_id'] - - # Shared post-processing context for modular lookups - track_title = metadata.get('title', '') - artist_name = metadata.get('album_artist', '') or metadata.get('artist', '') - - # Extract batch-level artist name for stable MB release cache keys. - # When downloading an album batch, all tracks should use the same artist key - # to guarantee they hit the same preflight-cached release MBID. - _track_info_for_pp = (context or {}).get('track_info', {}) or {} - _explicit_artist_for_pp = _track_info_for_pp.get('_explicit_artist_context') if isinstance(_track_info_for_pp, dict) else None - _batch_artist_name = None - if isinstance(_explicit_artist_for_pp, dict) and _explicit_artist_for_pp.get('name'): - _batch_artist_name = _explicit_artist_for_pp['name'] - elif isinstance(_explicit_artist_for_pp, str) and _explicit_artist_for_pp: - _batch_artist_name = _explicit_artist_for_pp - - pp = { - 'id_tags': id_tags, - 'track_title': track_title, - 'artist_name': artist_name, - 'batch_artist_name': _batch_artist_name, - 'metadata': metadata, - 'recording_mbid': None, - 'artist_mbid': None, - 'release_mbid': '', - 'mb_genres': [], - 'isrc': None, - 'deezer_bpm': None, 'deezer_isrc': None, - 'audiodb_mood': None, 'audiodb_style': None, 'audiodb_genre': None, - 'tidal_isrc': None, 'tidal_copyright': None, - 'qobuz_isrc': None, 'qobuz_copyright': None, 'qobuz_label': None, - 'lastfm_tags': [], 'lastfm_url': None, - 'genius_url': None, - 'release_year': None, # First source to find a year wins - } - - # Run each metadata source lookup in configured order - _pp_source_order = config_manager.get('metadata_enhancement.post_process_order', None) - if not _pp_source_order or not isinstance(_pp_source_order, list): - _pp_source_order = ['musicbrainz', 'deezer', 'audiodb', 'tidal', 'qobuz', 'lastfm', 'genius'] - - _pp_lookup_map = { - 'musicbrainz': _pp_lookup_musicbrainz, - 'deezer': _pp_lookup_deezer, - 'audiodb': _pp_lookup_audiodb, - 'tidal': _pp_lookup_tidal, - 'qobuz': _pp_lookup_qobuz, - 'lastfm': _pp_lookup_lastfm, - 'genius': _pp_lookup_genius, - } - - for source_name in _pp_source_order: - fn = _pp_lookup_map.get(source_name) - if fn: - fn(pp, _names_match) - - # Extract results from shared context after all lookups - recording_mbid = pp['recording_mbid'] - artist_mbid = pp['artist_mbid'] - _rc_mbid = pp['release_mbid'] - mb_genres = pp['mb_genres'] - isrc = pp['isrc'] - deezer_bpm = pp['deezer_bpm'] - deezer_isrc = pp['deezer_isrc'] - audiodb_mood = pp['audiodb_mood'] - audiodb_style = pp['audiodb_style'] - audiodb_genre = pp['audiodb_genre'] - tidal_isrc = pp['tidal_isrc'] - tidal_copyright = pp['tidal_copyright'] - qobuz_isrc = pp['qobuz_isrc'] - qobuz_copyright = pp['qobuz_copyright'] - qobuz_label = pp['qobuz_label'] - lastfm_tags = pp['lastfm_tags'] - lastfm_url = pp['lastfm_url'] - genius_url = pp['genius_url'] - id_tags = pp['id_tags'] - release_year = pp['release_year'] - - # If metadata already has a date from Spotify context, use that as fallback - if not release_year and metadata.get('date'): - yr = str(metadata['date'])[:4] - if yr.isdigit(): - release_year = yr - - # Store release MBID in metadata for downstream use (e.g. Cover Art Archive) - if _rc_mbid: - metadata['musicbrainz_release_id'] = _rc_mbid - - # Write release year to file tags if not already present - if release_year and 'ORIGINALDATE' not in id_tags: - id_tags['ORIGINALDATE'] = release_year - # If the file was written without a date tag, flag it for writing below - _needs_date_tag = release_year and not metadata.get('date') - if _needs_date_tag: - metadata['date'] = release_year - - # Update DB album year if currently missing - if release_year: - try: - _pp_album_name = metadata.get('album', '') - _pp_artist_name = metadata.get('album_artist', '') or metadata.get('artist', '') - if _pp_album_name and _pp_artist_name: - conn = get_database()._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - UPDATE albums SET year = ? - WHERE (year IS NULL OR year = 0) - AND id IN ( - SELECT al.id FROM albums al - JOIN artists ar ON ar.id = al.artist_id - WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) - ) - """, (int(release_year), _pp_album_name, _pp_artist_name)) - if cursor.rowcount > 0: - conn.commit() - logger.info(f"Updated album year to {release_year} in database") - else: - conn.rollback() - finally: - conn.close() - except Exception as e: - logger.error(f"Could not update album year in DB: {e}") - - # (All source lookups now handled by _pp_lookup_* functions called via configurable order above) - if False: # Dead code — old inline blocks preserved for reference during transition - try: - mb_service_for_detail = mb_worker.mb_service if mb_worker else None - if mb_service_for_detail: - with _mb_release_detail_cache_lock: - release_detail = _mb_release_detail_cache.get(_rc_mbid) - if release_detail is None: - release_detail = mb_service_for_detail.mb_client.get_release( - _rc_mbid, includes=['release-groups', 'labels', 'media', 'artist-credits', 'recordings'] - ) or {} - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[_rc_mbid] = release_detail - if release_detail: - rg = release_detail.get('release-group', {}) - if rg.get('id'): - id_tags['MUSICBRAINZ_RELEASEGROUPID'] = rg['id'] - ac = release_detail.get('artist-credit', []) - if ac and isinstance(ac[0], dict): - aa_artist = ac[0].get('artist', {}) - if aa_artist.get('id'): - id_tags['MUSICBRAINZ_ALBUMARTISTID'] = aa_artist['id'] - primary_type = rg.get('primary-type', '') - if primary_type: - id_tags['RELEASETYPE'] = primary_type - orig_date = rg.get('first-release-date', '') - if orig_date: - id_tags['ORIGINALDATE'] = orig_date - status = release_detail.get('status', '') - if status: - id_tags['RELEASESTATUS'] = status - country = release_detail.get('country', '') - if country: - id_tags['RELEASECOUNTRY'] = country - barcode = release_detail.get('barcode', '') - if barcode: - id_tags['BARCODE'] = barcode - media_list = release_detail.get('media', []) - if media_list: - media_format = media_list[0].get('format', '') - if media_format: - id_tags['MEDIA'] = media_format - id_tags['TOTALDISCS'] = str(len(media_list)) - label_info = release_detail.get('label-info', []) - if label_info and isinstance(label_info[0], dict): - cat_num = label_info[0].get('catalog-number', '') - if cat_num: - id_tags['CATALOGNUMBER'] = cat_num - text_rep = release_detail.get('text-representation', {}) - if isinstance(text_rep, dict) and text_rep.get('script'): - id_tags['SCRIPT'] = text_rep['script'] - asin = release_detail.get('asin', '') - if asin: - id_tags['ASIN'] = asin - # Release Track ID — match by disc + track position - _trk_num = metadata.get('track_number') - _disc_num = metadata.get('disc_number') or 1 - if _trk_num and media_list: - try: - _trk_num_int = int(_trk_num) - _disc_num_int = int(_disc_num) - for medium in media_list: - if medium.get('position', 1) == _disc_num_int: - for mtrack in (medium.get('tracks') or medium.get('track-list', [])): - if mtrack.get('position') == _trk_num_int and mtrack.get('id'): - id_tags['MUSICBRAINZ_RELEASETRACKID'] = mtrack['id'] - break - break - except (ValueError, TypeError): - pass - logger.info(f"MusicBrainz release details: type={primary_type or '?'}, " - f"country={country or '?'}, media={id_tags.get('MEDIA', '?')}") - except Exception as e: - logger.error(f"MusicBrainz release detail lookup failed (non-fatal): {e}") - - # ── 2b. Deezer lookup for BPM, ISRC fallback, and source IDs ── - deezer_bpm = None - deezer_isrc = None - if not config_manager.get('deezer.embed_tags', True): - pass - elif track_title and artist_name: - try: - dz_client = deezer_worker.client if deezer_worker else None - if dz_client: - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get('title', ''), track_title) and \ - _names_match(dz_result.get('artist', {}).get('name', ''), artist_name): - dz_track_id = dz_result['id'] - id_tags['DEEZER_TRACK_ID'] = str(dz_track_id) - dz_artist_id = dz_result.get('artist', {}).get('id') - if dz_artist_id: - id_tags['DEEZER_ARTIST_ID'] = str(dz_artist_id) - logger.info(f"Deezer track matched: {dz_track_id}") - - # Get full track details for BPM and ISRC - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get('bpm') - if bpm_val and bpm_val > 0: - deezer_bpm = bpm_val - dz_isrc = dz_details.get('isrc') - if dz_isrc: - deezer_isrc = dz_isrc - else: - logger.info("Deezer worker not available, skipping Deezer lookup") - except Exception as e: - logger.error(f"Deezer lookup failed (non-fatal): {e}") - - # ── 2c. AudioDB lookup for mood, style, genre, and source ID ── - audiodb_mood = None - audiodb_style = None - audiodb_genre = None - if not config_manager.get('audiodb.embed_tags', True): - pass - elif track_title and artist_name: - try: - adb_client = audiodb_worker.client if audiodb_worker else None - if adb_client: - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get('strTrack', ''), track_title) and \ - _names_match(adb_result.get('strArtist', ''), artist_name): - adb_track_id = adb_result.get('idTrack') - if adb_track_id: - id_tags['AUDIODB_TRACK_ID'] = str(adb_track_id) - logger.info(f"AudioDB track matched: {adb_track_id}") - # Use AudioDB's MusicBrainz IDs as fallbacks for any missing from MB lookup - adb_mb_track = adb_result.get('strMusicBrainzID') - if adb_mb_track and 'MUSICBRAINZ_RECORDING_ID' not in id_tags: - id_tags['MUSICBRAINZ_RECORDING_ID'] = adb_mb_track - recording_mbid = adb_mb_track - logger.warning(f"MusicBrainz recording ID from AudioDB fallback: {adb_mb_track}") - # NOTE: AudioDB's strMusicBrainzAlbumID is intentionally - # NOT used as a fallback for MUSICBRAINZ_RELEASE_ID. - # AudioDB links each track to its original album in MB, - # which differs per track on compilations and splits - # albums in players like Navidrome. Album MBID must come - # from match_release (cached) to stay consistent. - adb_mb_artist = adb_result.get('strMusicBrainzArtistID') - if adb_mb_artist and 'MUSICBRAINZ_ARTIST_ID' not in id_tags: - id_tags['MUSICBRAINZ_ARTIST_ID'] = adb_mb_artist - artist_mbid = adb_mb_artist - logger.warning(f"MusicBrainz artist ID from AudioDB fallback: {adb_mb_artist}") - audiodb_mood = adb_result.get('strMood') or None - audiodb_style = adb_result.get('strStyle') or None - audiodb_genre = adb_result.get('strGenre') or None - else: - logger.info("AudioDB worker not available, skipping AudioDB lookup") - except Exception as e: - logger.error(f"AudioDB lookup failed (non-fatal): {e}") - - # ── 2d. Tidal lookup for ISRC fallback, copyright, and source IDs ── - tidal_isrc = None - tidal_copyright = None - if not config_manager.get('tidal.embed_tags', True): - pass - elif track_title and artist_name: - try: - if tidal_client and tidal_client.is_authenticated(): - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get('title', ''), track_title): - td_track_id = td_result.get('id') - if td_track_id: - id_tags['TIDAL_TRACK_ID'] = str(td_track_id) - logger.info(f"Tidal track matched: {td_track_id}") - td_artist = td_result.get('artist', {}) - if isinstance(td_artist, dict) and td_artist.get('id'): - id_tags['TIDAL_ARTIST_ID'] = str(td_artist['id']) - # Get full details for ISRC and copyright - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - td_isrc = td_details.get('isrc') - if td_isrc: - tidal_isrc = td_isrc - td_copyright = td_details.get('copyright') - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get('text', td_copyright.get('name', '')) - if td_copyright: - tidal_copyright = td_copyright - except Exception as e: - logger.error(f"Tidal lookup failed (non-fatal): {e}") - - # ── 2e. Qobuz lookup for ISRC fallback, copyright, label, and source IDs ── - qobuz_isrc = None - qobuz_copyright = None - qobuz_label = None - if not config_manager.get('qobuz.embed_tags', True): - pass - elif track_title and artist_name: - try: - qz_client = qobuz_enrichment_worker.client if qobuz_enrichment_worker else None - if qz_client and qz_client.is_authenticated(): - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = (qz_result.get('performer') or {}) - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get('name', '') - if _names_match(qz_result.get('title', ''), track_title) and \ - _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get('id') - if qz_track_id: - id_tags['QOBUZ_TRACK_ID'] = str(qz_track_id) - logger.info(f"Qobuz track matched: {qz_track_id}") - if isinstance(qz_performer, dict) and qz_performer.get('id'): - id_tags['QOBUZ_ARTIST_ID'] = str(qz_performer['id']) - qz_isrc = qz_result.get('isrc') - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get('value', qz_isrc.get('id', '')) - if qz_isrc: - qobuz_isrc = qz_isrc - qz_copyright = qz_result.get('copyright') - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get('text', qz_copyright.get('name', '')) - if qz_copyright and isinstance(qz_copyright, str): - qobuz_copyright = qz_copyright - qz_album = qz_result.get('album', {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get('label', {}) - if isinstance(qz_label_info, dict) and qz_label_info.get('name'): - qobuz_label = qz_label_info['name'] - except Exception as e: - logger.error(f"Qobuz lookup failed (non-fatal): {e}") - - # ── 2f. Last.fm lookup for tags (genre merge) and URL ── - lastfm_tags = [] - lastfm_url = None - if not config_manager.get('lastfm.embed_tags', True): - pass - elif track_title and artist_name: - try: - lf_client = lastfm_worker.client if lastfm_worker else None - if lf_client: - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get('url') - if lf_url: - lastfm_url = lf_url - lf_toptags = lf_result.get('toptags', {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get('tag', []) - if isinstance(tag_list, list): - lastfm_tags = [t.get('name', '') for t in tag_list if isinstance(t, dict) and t.get('name')] - elif isinstance(tag_list, dict) and tag_list.get('name'): - lastfm_tags = [tag_list['name']] - logger.info(f"Last.fm track info found: {len(lastfm_tags)} tags") - except Exception as e: - logger.error(f"Last.fm lookup failed (non-fatal): {e}") - - # ── 2g. Genius lookup for source ID and URL ── - # Genius has an aggressive global rate limiter (30→60→120s backoff) that - # blocks ALL callers including post-processing. We check the backoff - # state directly and skip immediately if Genius is rate-limited, rather - # than entering search_song which would sleep for up to 120s. - genius_url = None - if not config_manager.get('genius.embed_tags', True): - pass - elif track_title and artist_name: - try: - import core.genius_client as _genius_module - if time.time() < _genius_module._rate_limit_until: - logger.info("Genius rate-limited, skipping (non-blocking)") - else: - g_client = genius_worker.client if genius_worker else None - if g_client: - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get('id') - if g_id: - id_tags['GENIUS_TRACK_ID'] = str(g_id) - logger.info(f"Genius song matched: {g_id}") - g_url = g_result.get('url') - if g_url: - genius_url = g_url - except Exception as e: - logger.error(f"Genius lookup failed (non-fatal): {e}") - - if not id_tags and not deezer_bpm and not deezer_isrc and not audiodb_mood and not audiodb_style: - return - - # ── 3. Filter tags by per-tag config, then write ── - filtered_tags = {} - for tag_name, value in id_tags.items(): - config_path = _TAG_CONFIG.get(tag_name) - if config_path and not _tag_enabled(config_path): - continue - filtered_tags[tag_name] = value - - # ── 3a. Write ID tags (MusicBrainz, source IDs, release info) ── - written = [] - - # Format-specific tag name mappings for Picard-compatible output - _ID3_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': ('UFID', 'http://musicbrainz.org'), - 'MUSICBRAINZ_ARTIST_ID': ('TXXX', 'MusicBrainz Artist Id'), - 'MUSICBRAINZ_RELEASE_ID': ('TXXX', 'MusicBrainz Album Id'), - 'MUSICBRAINZ_RELEASEGROUPID': ('TXXX', 'MusicBrainz Release Group Id'), - 'MUSICBRAINZ_ALBUMARTISTID': ('TXXX', 'MusicBrainz Album Artist Id'), - 'MUSICBRAINZ_RELEASETRACKID': ('TXXX', 'MusicBrainz Release Track Id'), - 'RELEASETYPE': ('TXXX', 'MusicBrainz Album Type'), - 'RELEASESTATUS': ('TXXX', 'MusicBrainz Album Status'), - 'RELEASECOUNTRY': ('TXXX', 'MusicBrainz Album Release Country'), - 'ORIGINALDATE': ('TDOR', None), - 'MEDIA': ('TMED', None), - } - _VORBIS_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': 'MUSICBRAINZ_TRACKID', - 'MUSICBRAINZ_ARTIST_ID': 'MUSICBRAINZ_ARTISTID', - 'MUSICBRAINZ_RELEASE_ID': 'MUSICBRAINZ_ALBUMID', - 'MUSICBRAINZ_RELEASEGROUPID': 'MUSICBRAINZ_RELEASEGROUPID', - 'MUSICBRAINZ_ALBUMARTISTID': 'MUSICBRAINZ_ALBUMARTISTID', - 'MUSICBRAINZ_RELEASETRACKID': 'MUSICBRAINZ_RELEASETRACKID', - } - _MP4_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': 'MusicBrainz Track Id', - 'MUSICBRAINZ_ARTIST_ID': 'MusicBrainz Artist Id', - 'MUSICBRAINZ_RELEASE_ID': 'MusicBrainz Album Id', - 'MUSICBRAINZ_RELEASEGROUPID': 'MusicBrainz Release Group Id', - 'MUSICBRAINZ_ALBUMARTISTID': 'MusicBrainz Album Artist Id', - 'MUSICBRAINZ_RELEASETRACKID': 'MusicBrainz Release Track Id', - 'RELEASETYPE': 'MusicBrainz Album Type', - 'RELEASESTATUS': 'MusicBrainz Album Status', - 'RELEASECOUNTRY': 'MusicBrainz Album Release Country', - } - - # MP3 (ID3) - if isinstance(audio_file.tags, ID3): - for tag_name, value in filtered_tags.items(): - id3_spec = _ID3_TAG_MAP.get(tag_name) - if id3_spec: - frame_type, desc = id3_spec - if frame_type == 'UFID': - audio_file.tags.add(UFID(owner=desc, data=value.encode('ascii'))) - written.append(f'UFID:{desc}') - elif frame_type == 'TDOR': - audio_file.tags.add(TDOR(encoding=3, text=[value])) - written.append('TDOR') - elif frame_type == 'TMED': - audio_file.tags.add(TMED(encoding=3, text=[value])) - written.append('TMED') - else: # TXXX - audio_file.tags.add(TXXX(encoding=3, desc=desc, text=[value])) - written.append(f'TXXX:{desc}') - else: - audio_file.tags.add(TXXX(encoding=3, desc=tag_name, text=[str(value)])) - written.append(f'TXXX:{tag_name}') - - # FLAC / OGG Vorbis - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - for tag_name, value in filtered_tags.items(): - vorbis_key = _VORBIS_TAG_MAP.get(tag_name, tag_name) - audio_file[vorbis_key] = [str(value)] - written.append(vorbis_key) - - # MP4 (M4A/AAC) - elif isinstance(audio_file, MP4): - for tag_name, value in filtered_tags.items(): - mp4_desc = _MP4_TAG_MAP.get(tag_name, tag_name) - key = f'----:com.apple.iTunes:{mp4_desc}' - audio_file[key] = [MP4FreeForm(str(value).encode('utf-8'))] - written.append(key) - - if written: - logger.info(f"Embedded IDs: {', '.join(written)}") - - # ── 3a½. Write date tag if discovered during lookups (initial write had no date) ── - if _needs_date_tag and release_year: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TDRC(encoding=3, text=[release_year])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['date'] = [release_year] - elif isinstance(audio_file, MP4): - audio_file['\xa9day'] = [release_year] - logger.info(f"Date tag: {release_year}") - - # ── 3b. Write BPM tag (from Deezer) ── - if _tag_enabled('deezer.tags.bpm') and deezer_bpm and deezer_bpm > 0: - bpm_int = int(deezer_bpm) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TBPM(encoding=3, text=[str(bpm_int)])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['BPM'] = [str(bpm_int)] - elif isinstance(audio_file, MP4): - audio_file['tmpo'] = [bpm_int] - logger.info(f"BPM: {bpm_int}") - - # ── 3c. Write mood tag (from AudioDB) ── - if _tag_enabled('audiodb.tags.mood') and audiodb_mood: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='MOOD', text=[audiodb_mood])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['MOOD'] = [audiodb_mood] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:MOOD'] = [MP4FreeForm(audiodb_mood.encode('utf-8'))] - logger.info(f"Mood: {audiodb_mood}") - - # ── 3d. Write style tag (from AudioDB) ── - if _tag_enabled('audiodb.tags.style') and audiodb_style: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='STYLE', text=[audiodb_style])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['STYLE'] = [audiodb_style] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:STYLE'] = [MP4FreeForm(audiodb_style.encode('utf-8'))] - logger.info(f"Style: {audiodb_style}") - - # ── 4. Merge genres (Spotify + MusicBrainz + AudioDB + Last.fm) and overwrite tag ── - if _tag_enabled('metadata_enhancement.tags.genre_merge'): - enrichment_genres = (mb_genres if _tag_enabled('musicbrainz.tags.genres') else []) + \ - ([audiodb_genre] if audiodb_genre and _tag_enabled('audiodb.tags.genre') else []) + \ - (lastfm_tags if _tag_enabled('lastfm.tags.genres') else []) - if enrichment_genres: - from core.genre_filter import filter_genres as _gf - enrichment_genres = _gf(enrichment_genres, config_manager) - spotify_genres = [g.strip() for g in metadata.get('genre', '').split(',') if g.strip()] - seen = set() - merged = [] - for g in spotify_genres + enrichment_genres: - key = g.strip().lower() - if key and key not in seen: - seen.add(key) - merged.append(g.strip().title()) - if len(merged) >= 5: - break - - if merged: - genre_string = ', '.join(merged) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TCON(encoding=3, text=[genre_string])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['GENRE'] = [genre_string] - elif isinstance(audio_file, MP4): - audio_file['\xa9gen'] = [genre_string] - logger.info(f"Genres merged: {genre_string}") - - # ── 5. Write ISRC if available (per-source fallback chain) ── - _isrc_candidates = [] - if isrc and _tag_enabled('musicbrainz.tags.isrc'): - _isrc_candidates.append(('MusicBrainz', isrc)) - if deezer_isrc and _tag_enabled('deezer.tags.isrc'): - _isrc_candidates.append(('Deezer', deezer_isrc)) - if tidal_isrc and _tag_enabled('tidal.tags.isrc'): - _isrc_candidates.append(('Tidal', tidal_isrc)) - if qobuz_isrc and _tag_enabled('qobuz.tags.isrc'): - _isrc_candidates.append(('Qobuz', qobuz_isrc)) - if _isrc_candidates: - source, final_isrc = _isrc_candidates[0] - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TSRC(encoding=3, text=[final_isrc])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['ISRC'] = [final_isrc] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:ISRC'] = [MP4FreeForm(final_isrc.encode('utf-8'))] - logger.info(f"ISRC ({source}): {final_isrc}") - - # ── 6. Write copyright tag (Tidal → Qobuz fallback) ── - _copyright_candidates = [] - if tidal_copyright and _tag_enabled('tidal.tags.copyright'): - _copyright_candidates.append(('Tidal', tidal_copyright)) - if qobuz_copyright and _tag_enabled('qobuz.tags.copyright'): - _copyright_candidates.append(('Qobuz', qobuz_copyright)) - if _copyright_candidates: - source, final_copyright = _copyright_candidates[0] - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TCOP(encoding=3, text=[final_copyright])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['COPYRIGHT'] = [final_copyright] - elif isinstance(audio_file, MP4): - audio_file['cprt'] = [final_copyright] - logger.info(f"©️ Copyright ({source}): {final_copyright[:60]}") - - # ── 7. Write label/publisher tag (from Qobuz) ── - if _tag_enabled('qobuz.tags.label') and qobuz_label: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TPUB(encoding=3, text=[qobuz_label])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['LABEL'] = [qobuz_label] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:LABEL'] = [MP4FreeForm(qobuz_label.encode('utf-8'))] - logger.info(f"Label (Qobuz): {qobuz_label}") - - # ── 8. Write Last.fm and Genius URLs as custom tags ── - if _tag_enabled('lastfm.tags.url') and lastfm_url: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='LASTFM_URL', text=[lastfm_url])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['LASTFM_URL'] = [lastfm_url] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:LASTFM_URL'] = [MP4FreeForm(lastfm_url.encode('utf-8'))] - - if _tag_enabled('genius.tags.url') and genius_url: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='GENIUS_URL', text=[genius_url])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['GENIUS_URL'] = [genius_url] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:GENIUS_URL'] = [MP4FreeForm(genius_url.encode('utf-8'))] - - except Exception as e: - logger.error(f"Error embedding source IDs (non-fatal): {e}") - def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): - """Downloads cover.jpg into the specified directory. - Tries Cover Art Archive first (high-res) if MusicBrainz release ID is available - (set by _enhance_file_metadata during post-processing), falls back to source URL. - Accepts optional context to extract image URL from spotify_album when album_info lacks it.""" - if not config_manager.get('metadata_enhancement.cover_art_download', True): - return - try: - cover_path = os.path.join(target_dir, "cover.jpg") - release_mbid = album_info.get('musicbrainz_release_id') if album_info else None - prefer_caa = config_manager.get('metadata_enhancement.prefer_caa_art', False) - - # If cover.jpg exists but we now have a CAA MBID, check if we should upgrade - if os.path.exists(cover_path): - if release_mbid and prefer_caa: - try: - existing_size = os.path.getsize(cover_path) - # Typical Spotify/iTunes cover is ~50-150KB at 640x640 - # CAA covers are usually 300KB+ at 1200x1200 - if existing_size > 200_000: - return # Already high-res, skip - # Low-res cover exists — try to upgrade from CAA - is_upgrade = True - logger.info(f"Existing cover.jpg is {existing_size // 1024}KB — attempting CAA upgrade...") - except Exception: - return - else: - return - else: - is_upgrade = False - - image_data = None - - # Try Cover Art Archive first (often 1200x1200+, original quality) — opt-in - # The MBID is stored in album_info by _enhance_file_metadata before this is called - if release_mbid and prefer_caa: - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={'Accept': 'image/*'}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - if image_data and len(image_data) > 1000: - logger.info(f"Cover art from Cover Art Archive ({len(image_data) // 1024}KB)") - else: - image_data = None - except Exception: - image_data = None - - # If upgrading and CAA failed, keep existing cover — don't overwrite with same low-res - if is_upgrade and not image_data: - logger.error("CAA upgrade failed — keeping existing cover.jpg") - return - - # Fallback to Spotify/iTunes/Deezer URL - if not image_data: - art_url = album_info.get('album_image_url') - # If album_info lacks the URL, try the context's spotify_album - if not art_url and context: - spotify_album = context.get('spotify_album') or {} - art_url = spotify_album.get('image_url') - # Also try images array (raw Spotify API format) - if not art_url: - images = spotify_album.get('images', []) - if images and isinstance(images, list) and len(images) > 0: - art_url = images[0].get('url') if isinstance(images[0], dict) else None - if art_url: - logger.info("Using cover art URL from spotify_album context") - # Upgrade to highest available resolution before fetching - if art_url and 'i.scdn.co' in art_url: - from core.spotify_client import _upgrade_spotify_image_url - art_url = _upgrade_spotify_image_url(art_url) - elif art_url and 'mzstatic.com' in art_url: - import re as _re - art_url = _re.sub(r'\d+x\d+bb', '3000x3000bb', art_url) - if not art_url: - logger.warning("No cover art URL available for download.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - - if not image_data: - return - - with open(cover_path, 'wb') as f: - f.write(image_data) - - logger.info(f"Cover art downloaded to: {cover_path}") - except Exception as e: - logger.error(f"Error downloading cover.jpg: {e}") - - - + return metadata_enrichment.download_cover_art( + album_info, + target_dir, + context, + ) def _get_spotify_album_tracks(spotify_album: dict) -> list: """Fetches all tracks for a given Spotify album ID.""" @@ -20568,695 +18192,29 @@ def _match_track_to_spotify_title(slsk_track_meta: dict, spotify_tracks: list) - return slsk_track_meta # Fallback to original - -# --- Post-Processing Logic --- -# ── Modular post-processing metadata lookup functions ── -# Each function receives a shared `pp` context dict and writes its results to it. -# The orchestrator in _post_process_matched_download calls them in configurable order. - -def _pp_lookup_musicbrainz(pp, _names_match): - """MusicBrainz: recording MBID, artist MBID, release MBID, ISRC, genres, release details.""" - if not config_manager.get('musicbrainz.embed_tags', True): - return - track_title = pp['track_title'] - artist_name = pp['artist_name'] - metadata = pp['metadata'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - mb_service = mb_worker.mb_service if mb_worker else None - if not mb_service: - logger.info("MusicBrainz worker not available, skipping MBID lookup") - return - result = mb_service.match_recording(track_title, artist_name) - if result and result.get('mbid'): - pp['recording_mbid'] = result['mbid'] - id_tags['MUSICBRAINZ_RECORDING_ID'] = pp['recording_mbid'] - logger.info(f"MusicBrainz recording matched: {pp['recording_mbid']}") - details = mb_service.mb_client.get_recording(pp['recording_mbid'], includes=['isrcs', 'genres']) - if details: - isrcs = details.get('isrcs', []) - if isrcs: - pp['isrc'] = isrcs[0] - pp['mb_genres'] = [g['name'] for g in sorted(details.get('genres', []), key=lambda x: x.get('count', 0), reverse=True)] - - track_artist_name = metadata.get('artist', '') or artist_name - if ', ' in track_artist_name: - track_artist_name = track_artist_name.split(', ')[0] - artist_result = mb_service.match_artist(track_artist_name) - if artist_result and artist_result.get('mbid'): - pp['artist_mbid'] = artist_result['mbid'] - id_tags['MUSICBRAINZ_ARTIST_ID'] = pp['artist_mbid'] - - album_name_for_mb = metadata.get('album', '') - if album_name_for_mb: - # Use normalized key (strips edition suffixes) so "Album (Deluxe Edition)" - # and "Album (Deluxe)" and "Album" all share the same cached MBID. - # This prevents Navidrome from splitting one album into multiple entries. - # Prefer batch-level artist name (from album download context) for the cache key - # so all tracks in the batch hit the same preflight-cached entry, even if - # per-track metadata spells the artist slightly differently. - _artist_key = (pp.get('batch_artist_name') or artist_name).lower().strip() - _rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), _artist_key) - _rc_key_exact = (album_name_for_mb.lower().strip(), _artist_key) - with _mb_release_cache_lock: - # Check normalized key first (catches edition variants) - cached = _mb_release_cache.get(_rc_key_norm) - if cached is None: - cached = _mb_release_cache.get(_rc_key_exact) - if cached is not None: - pp['release_mbid'] = cached - else: - try: - _rc_result = mb_service.match_release(album_name_for_mb, artist_name) - pp['release_mbid'] = _rc_result.get('mbid', '') if _rc_result else '' - except Exception: - pp['release_mbid'] = '' - # Cache under both normalized and exact keys - _mb_release_cache[_rc_key_norm] = pp['release_mbid'] - _mb_release_cache[_rc_key_exact] = pp['release_mbid'] - if pp['release_mbid']: - id_tags['MUSICBRAINZ_RELEASE_ID'] = pp['release_mbid'] - - # Release details (group, barcode, media, etc.) - if pp['release_mbid']: - with _mb_release_detail_cache_lock: - release_detail = _mb_release_detail_cache.get(pp['release_mbid']) - if release_detail is None: - release_detail = mb_service.mb_client.get_release( - pp['release_mbid'], includes=['release-groups', 'labels', 'media', 'artist-credits', 'recordings'] - ) or {} - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[pp['release_mbid']] = release_detail - if release_detail: - rg = release_detail.get('release-group', {}) - if rg.get('id'): id_tags['MUSICBRAINZ_RELEASEGROUPID'] = rg['id'] - ac = release_detail.get('artist-credit', []) - if ac and isinstance(ac[0], dict): - aa = ac[0].get('artist', {}) - if aa.get('id'): id_tags['MUSICBRAINZ_ALBUMARTISTID'] = aa['id'] - if rg.get('primary-type'): id_tags['RELEASETYPE'] = rg['primary-type'] - if rg.get('first-release-date'): - id_tags['ORIGINALDATE'] = rg['first-release-date'] - if not pp['release_year'] and len(rg['first-release-date']) >= 4: - yr = rg['first-release-date'][:4] - if yr.isdigit(): - pp['release_year'] = yr - if release_detail.get('status'): id_tags['RELEASESTATUS'] = release_detail['status'] - if release_detail.get('country'): id_tags['RELEASECOUNTRY'] = release_detail['country'] - if release_detail.get('barcode'): id_tags['BARCODE'] = release_detail['barcode'] - media_list = release_detail.get('media', []) - if media_list: - fmt = media_list[0].get('format', '') - if fmt: id_tags['MEDIA'] = fmt - id_tags['TOTALDISCS'] = str(len(media_list)) - label_info = release_detail.get('label-info', []) - if label_info and isinstance(label_info[0], dict): - cat = label_info[0].get('catalog-number', '') - if cat: id_tags['CATALOGNUMBER'] = cat - text_rep = release_detail.get('text-representation', {}) - if isinstance(text_rep, dict) and text_rep.get('script'): - id_tags['SCRIPT'] = text_rep['script'] - if release_detail.get('asin'): id_tags['ASIN'] = release_detail['asin'] - # Picard-style: pull recording MBID from the release tracklist - # instead of using the independent match_recording() result. - # This guarantees the recording ID is consistent with the release. - _trk_num = metadata.get('track_number') - _disc_num = metadata.get('disc_number') or 1 - if _trk_num and media_list: - try: - _trk_num_int, _disc_num_int = int(_trk_num), int(_disc_num) - for medium in media_list: - if medium.get('position', 1) == _disc_num_int: - for mtrack in (medium.get('tracks') or medium.get('track-list', [])): - if mtrack.get('position') == _trk_num_int: - if mtrack.get('id'): - id_tags['MUSICBRAINZ_RELEASETRACKID'] = mtrack['id'] - # Override recording MBID with the one from this release's tracklist - _release_recording = mtrack.get('recording', {}) - if _release_recording.get('id'): - pp['recording_mbid'] = _release_recording['id'] - id_tags['MUSICBRAINZ_RECORDING_ID'] = _release_recording['id'] - logger.info(f"MusicBrainz recording from release tracklist: {_release_recording['id']}") - break - break - except (ValueError, TypeError): - pass - except Exception as e: - logger.error(f"MusicBrainz lookup failed (non-fatal): {e}") - - -def _pp_lookup_deezer(pp, _names_match): - """Deezer: BPM, ISRC fallback, track/artist IDs.""" - if not config_manager.get('deezer.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - dz_client = deezer_worker.client if deezer_worker else None - if not dz_client: - logger.info("Deezer worker not available, skipping Deezer lookup") - return - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get('title', ''), track_title) and \ - _names_match(dz_result.get('artist', {}).get('name', ''), artist_name): - dz_track_id = dz_result['id'] - id_tags['DEEZER_TRACK_ID'] = str(dz_track_id) - dz_artist_id = dz_result.get('artist', {}).get('id') - if dz_artist_id: - id_tags['DEEZER_ARTIST_ID'] = str(dz_artist_id) - logger.info(f"Deezer track matched: {dz_track_id}") - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get('bpm') - if bpm_val and bpm_val > 0: - pp['deezer_bpm'] = bpm_val - dz_isrc = dz_details.get('isrc') - if dz_isrc: - pp['deezer_isrc'] = dz_isrc - # Release year from Deezer album - if not pp['release_year']: - dz_album = dz_result.get('album', {}) - dz_release = (dz_album.get('release_date', '') if isinstance(dz_album, dict) else '') or '' - if len(dz_release) >= 4 and dz_release[:4].isdigit(): - pp['release_year'] = dz_release[:4] - except Exception as e: - logger.error(f"Deezer lookup failed (non-fatal): {e}") - - -def _pp_lookup_audiodb(pp, _names_match): - """AudioDB: mood, style, genre, track ID, MusicBrainz ID fallbacks.""" - if not config_manager.get('audiodb.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - adb_client = audiodb_worker.client if audiodb_worker else None - if not adb_client: - logger.info("AudioDB worker not available, skipping AudioDB lookup") - return - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get('strTrack', ''), track_title) and \ - _names_match(adb_result.get('strArtist', ''), artist_name): - adb_track_id = adb_result.get('idTrack') - if adb_track_id: - id_tags['AUDIODB_TRACK_ID'] = str(adb_track_id) - logger.info(f"AudioDB track matched: {adb_track_id}") - adb_mb_track = adb_result.get('strMusicBrainzID') - if adb_mb_track and 'MUSICBRAINZ_RECORDING_ID' not in id_tags: - id_tags['MUSICBRAINZ_RECORDING_ID'] = adb_mb_track - pp['recording_mbid'] = adb_mb_track - logger.warning(f"MusicBrainz recording ID from AudioDB fallback: {adb_mb_track}") - adb_mb_artist = adb_result.get('strMusicBrainzArtistID') - if adb_mb_artist and 'MUSICBRAINZ_ARTIST_ID' not in id_tags: - id_tags['MUSICBRAINZ_ARTIST_ID'] = adb_mb_artist - pp['artist_mbid'] = adb_mb_artist - logger.warning(f"MusicBrainz artist ID from AudioDB fallback: {adb_mb_artist}") - pp['audiodb_mood'] = adb_result.get('strMood') or None - pp['audiodb_style'] = adb_result.get('strStyle') or None - pp['audiodb_genre'] = adb_result.get('strGenre') or None - except Exception as e: - logger.error(f"AudioDB lookup failed (non-fatal): {e}") - - -def _pp_lookup_tidal(pp, _names_match): - """Tidal: ISRC fallback, copyright, track/artist IDs.""" - if not config_manager.get('tidal.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - if not (tidal_client and tidal_client.is_authenticated()): - return - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get('title', ''), track_title): - td_track_id = td_result.get('id') - if td_track_id: - id_tags['TIDAL_TRACK_ID'] = str(td_track_id) - logger.info(f"Tidal track matched: {td_track_id}") - td_artist = td_result.get('artist', {}) - if isinstance(td_artist, dict) and td_artist.get('id'): - id_tags['TIDAL_ARTIST_ID'] = str(td_artist['id']) - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - td_isrc = td_details.get('isrc') - if td_isrc: - pp['tidal_isrc'] = td_isrc - td_copyright = td_details.get('copyright') - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get('text', td_copyright.get('name', '')) - if td_copyright: - pp['tidal_copyright'] = td_copyright - # Release year from Tidal album - if not pp['release_year']: - td_album = td_result.get('album', {}) - td_release = '' - if isinstance(td_album, dict): - td_release = str(td_album.get('release_date', '') or td_album.get('releaseDate', '') or '') - if len(td_release) >= 4 and td_release[:4].isdigit(): - pp['release_year'] = td_release[:4] - except Exception as e: - logger.error(f"Tidal lookup failed (non-fatal): {e}") - - -def _pp_lookup_qobuz(pp, _names_match): - """Qobuz: ISRC fallback, copyright, label, track/artist IDs.""" - if not config_manager.get('qobuz.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - qz_client = qobuz_enrichment_worker.client if qobuz_enrichment_worker else None - if not (qz_client and qz_client.is_authenticated()): - return - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = (qz_result.get('performer') or {}) - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get('name', '') - if _names_match(qz_result.get('title', ''), track_title) and \ - _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get('id') - if qz_track_id: - id_tags['QOBUZ_TRACK_ID'] = str(qz_track_id) - logger.info(f"Qobuz track matched: {qz_track_id}") - if isinstance(qz_performer, dict) and qz_performer.get('id'): - id_tags['QOBUZ_ARTIST_ID'] = str(qz_performer['id']) - qz_isrc = qz_result.get('isrc') - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get('value', qz_isrc.get('id', '')) - if qz_isrc: - pp['qobuz_isrc'] = qz_isrc - qz_copyright = qz_result.get('copyright') - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get('text', qz_copyright.get('name', '')) - if qz_copyright and isinstance(qz_copyright, str): - pp['qobuz_copyright'] = qz_copyright - qz_album = qz_result.get('album', {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get('label', {}) - if isinstance(qz_label_info, dict) and qz_label_info.get('name'): - pp['qobuz_label'] = qz_label_info['name'] - # Release year from Qobuz album (prefer date string over Unix timestamp) - if not pp['release_year']: - qz_release = str(qz_album.get('release_date_original', '') or '') - if not qz_release: - # released_at is a Unix timestamp — convert to year - qz_ts = qz_album.get('released_at') - if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: - import datetime as _dt - qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) - if len(qz_release) >= 4 and qz_release[:4].isdigit(): - pp['release_year'] = qz_release[:4] - except Exception as e: - logger.error(f"Qobuz lookup failed (non-fatal): {e}") - - -def _pp_lookup_lastfm(pp, _names_match): - """Last.fm: genre tags, track URL.""" - if not config_manager.get('lastfm.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - if not track_title or not artist_name: - return - try: - lf_client = lastfm_worker.client if lastfm_worker else None - if not lf_client: - return - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get('url') - if lf_url: - pp['lastfm_url'] = lf_url - lf_toptags = lf_result.get('toptags', {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get('tag', []) - if isinstance(tag_list, list): - pp['lastfm_tags'] = [t.get('name', '') for t in tag_list if isinstance(t, dict) and t.get('name')] - elif isinstance(tag_list, dict) and tag_list.get('name'): - pp['lastfm_tags'] = [tag_list['name']] - logger.info(f"Last.fm track info found: {len(pp['lastfm_tags'])} tags") - except Exception as e: - logger.error(f"Last.fm lookup failed (non-fatal): {e}") - - -def _pp_lookup_genius(pp, _names_match): - """Genius: track ID, URL.""" - if not config_manager.get('genius.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - import core.genius_client as _genius_module - if time.time() < _genius_module._rate_limit_until: - logger.info("Genius rate-limited, skipping (non-blocking)") - return - g_client = genius_worker.client if genius_worker else None - if not g_client: - return - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get('id') - if g_id: - id_tags['GENIUS_TRACK_ID'] = str(g_id) - logger.info(f"Genius song matched: {g_id}") - g_url = g_result.get('url') - if g_url: - pp['genius_url'] = g_url - except Exception as e: - logger.error(f"Genius lookup failed (non-fatal): {e}") - - def _post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id): """ NEW VERIFICATION WORKFLOW: Enhanced post-processing with file verification. Only sets task status to 'completed' after successful file verification and move operation. """ - _pp = pp_logger - try: - # Call the existing post-processing logic (but skip its completion callback) - # We'll handle the completion callback ourselves after verification - original_task_id = context.pop('task_id', None) # Temporarily remove to prevent double callback - original_batch_id = context.pop('batch_id', None) - _post_process_matched_download(context_key, context, file_path) - # Restore the IDs for our own callback - if original_task_id: - context['task_id'] = original_task_id - if original_batch_id: - context['batch_id'] = original_batch_id + from core.import_pipeline import post_process_matched_download_with_verification + return post_process_matched_download_with_verification( + context_key, + context, + file_path, + task_id, + batch_id, + _build_import_pipeline_runtime(), + ) - # Check if race guard detected the source file was already gone - if context.get('_race_guard_failed'): - _pp.info(f"Race guard: source file gone for task {task_id} — marking as failed") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task' - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=False) - return - - # Check if AcoustID quarantined the file — no further processing needed - if context.get('_acoustid_quarantined'): - failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed') - _pp.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}" - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=False) - return - - # Check if simple download handler already completed everything - if context.get('_simple_download_completed'): - expected_final_path = context.get('_final_path') - - if expected_final_path and os.path.exists(expected_final_path): - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=True) - return - else: - _pp.info(f"FAILED simple download file not found at: {expected_final_path} (task={task_id}, context={context_key})") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f'Downloaded file not found at expected location: {os.path.basename(expected_final_path)}' - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - return - - # VERIFICATION: Use the actual path that _post_process_matched_download computed and - # moved the file to, instead of recomputing independently. Independent recomputation - # can produce path mismatches (e.g., track_number extraction/validation differences) - # causing false "file verification failed" errors on successfully processed files. - expected_final_path = context.get('_final_processed_path') - if not expected_final_path: - _pp.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success") - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=True) - return - - # VERIFICATION: Check if file exists at the path processing actually used - if os.path.exists(expected_final_path): - # Mark task as completed only after successful verification - redownload_ctx = None - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - download_tasks[task_id]['metadata_enhanced'] = True - redownload_ctx = download_tasks[task_id].get('_redownload_context') - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Redownload hook: delete old file and update DB path (outside locks) - if redownload_ctx: - try: - old_path = redownload_ctx.get('old_file_path') - lib_track_id = redownload_ctx.get('library_track_id') - if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path): - if os.path.normpath(old_path) != os.path.normpath(expected_final_path): - os.remove(old_path) - logger.info(f"[Redownload] Deleted old file: {old_path}") - if lib_track_id and expected_final_path: - _rd_db = get_database() - _rd_conn = _rd_db._get_connection() - _rd_cursor = _rd_conn.cursor() - _rd_cursor.execute(""" - UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP - WHERE id = ? - """, (expected_final_path, lib_track_id)) - _rd_conn.commit() - _rd_conn.close() - logger.info(f"[Redownload] Updated DB path for track {lib_track_id}") - except Exception as e: - logger.error(f"[Redownload] Post-processing hook error: {e}") - - _on_download_completed(batch_id, task_id, success=True) - else: - # Log failure details for diagnosis - track_name = context.get('original_search_result', {}).get('spotify_clean_title', context_key) - _pp.info(f"FAILED verification for '{track_name}' (task={task_id})") - _pp.info(f" expected_final_path: {expected_final_path}") - _pp.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}") - _pp.info(f" is_album={context.get('is_album_download', False)}, has_clean_data={context.get('has_clean_spotify_data', False)}") - expected_dir = os.path.dirname(expected_final_path) - if os.path.exists(expected_dir): - dir_contents = os.listdir(expected_dir) - _pp.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}") - else: - _pp.info(f" directory does not exist: {expected_dir}") - - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing' - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - - except Exception as e: - import traceback - _pp.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}") - _pp.info(traceback.format_exc()) - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}" - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - - -def _check_flac_bit_depth(file_path, context, context_key): - """ - Check if a FLAC file matches the user's preferred bit depth. - Returns True if the file was rejected (caller should return), False if OK. - With fallback enabled (default), accepts any FLAC bit depth rather than quarantining. - """ - if not context.get('_audio_quality', '').startswith('FLAC'): - return False - - from database.music_database import MusicDatabase - _qp_db = MusicDatabase() - _quality_profile = _qp_db.get_quality_profile() - _flac_config = _quality_profile.get('qualities', {}).get('flac', {}) - _flac_pref = _flac_config.get('bit_depth', 'any') - - if _flac_pref == 'any': - return False - - # Parse actual bit depth from quality string like "FLAC 16bit" - _actual_bits = context['_audio_quality'].replace('FLAC ', '').replace('bit', '') - if _actual_bits == _flac_pref: - return False - - # Bit depth doesn't match preference — check if fallback or downsample is enabled - _flac_fallback = _flac_config.get('bit_depth_fallback', True) - _downsample_enabled = config_manager.get('lossy_copy.downsample_hires', False) - - if _flac_fallback or _downsample_enabled: - # Accept the file — it will be downsampled or a FLAC at any bit depth is better than a failed download - track_info = context.get('track_info', {}) - track_name = track_info.get('name', os.path.basename(file_path)) - if _downsample_enabled: - logger.info(f"[FLAC Downsample] Accepted {_actual_bits}-bit FLAC (will be downsampled to {_flac_pref}-bit): {track_name}") - else: - logger.warning(f"[FLAC Fallback] Accepted {_actual_bits}-bit FLAC (preferred {_flac_pref}-bit): {track_name}") - return False - - # Strict mode — reject and quarantine - rejection_msg = f"FLAC bit depth mismatch: file is {_actual_bits}-bit, preference is {_flac_pref}-bit" - try: - quarantine_path = _move_to_quarantine(file_path, context, rejection_msg) - logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") - except Exception as quarantine_error: - logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") - try: - os.remove(file_path) - except Exception: - pass - - context['_bitdepth_rejected'] = True - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_msg}" - if task_id and batch_id: - _on_download_completed(batch_id, task_id, success=False) - return True - - -def _move_to_quarantine(file_path: str, context: dict, reason: str) -> str: - """ - Move a file to quarantine folder when AcoustID verification fails. - Creates a JSON sidecar file with metadata about why the file was quarantined. - - Args: - file_path: Original file path - context: Download context with track info - reason: Reason for quarantine - - Returns: - Path to quarantined file - """ - import json - from pathlib import Path - from datetime import datetime - - # Get quarantine directory (inside download folder — always writable, even in Docker) - download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - quarantine_dir = Path(download_dir) / "ss_quarantine" - quarantine_dir.mkdir(parents=True, exist_ok=True) - - # Create quarantine entry with timestamp - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - original_name = Path(file_path).stem - file_ext = Path(file_path).suffix - - # Build quarantine filename: TIMESTAMP_originalname.ext.quarantined - # The .quarantined extension prevents audio file searches and media servers - # from picking up known-wrong files sitting in the downloads folder. - quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined" - quarantine_path = quarantine_dir / quarantine_filename - - # Move file to quarantine - _safe_move_file(file_path, str(quarantine_path)) - - # Write metadata sidecar file - metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json" - - # Extract track info from context - track_info = context.get('track_info', {}) - original_search = context.get('original_search_result', {}) - spotify_artist = context.get('spotify_artist', {}) - - metadata = { - 'original_filename': Path(file_path).name, - 'quarantine_reason': reason, - 'timestamp': datetime.now().isoformat(), - 'expected_track': ( - original_search.get('spotify_clean_title') or - track_info.get('name') or - original_search.get('title', 'Unknown') - ), - 'expected_artist': spotify_artist.get('name', 'Unknown'), - 'context_key': context.get('context_key', 'unknown') - } - - try: - with open(metadata_path, 'w', encoding='utf-8') as f: - json.dump(metadata, f, indent=2, ensure_ascii=False) - except Exception as e: - logger.warning(f"Failed to write quarantine metadata: {e}") - - logger.warning(f"File quarantined: {quarantine_path} - Reason: {reason}") - - try: - if automation_engine: - ti = context.get('track_info', {}) - artists = ti.get('artists', []) - artist_name = '' - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - automation_engine.emit('download_quarantined', { - 'artist': artist_name, - 'title': ti.get('name', ''), - 'reason': reason or 'Unknown', - }) - except Exception: - pass - - return str(quarantine_path) +def _build_import_pipeline_runtime(): + """Collect live controller dependencies for the shared import pipeline.""" + return types.SimpleNamespace( + automation_engine=automation_engine, + on_download_completed=_on_download_completed, + web_scan_manager=web_scan_manager, + repair_worker=repair_worker, + ) def _safe_move_file(src, dst): @@ -21362,872 +18320,8 @@ def _post_process_matched_download(context_key, context, file_path): Also handles simple downloads (from search page "Download" button) which just move files to /Transfer without metadata enhancement. """ - # --- PER-FILE LOCK --- - # Acquire a per-context-key lock so only one thread processes a given file at a time. - # The Stream Processor and Verification Worker both call this function for the same file. - # Without serialization, they race to move the source file and the loser gets FileNotFoundError. - # With the lock, the second thread waits, then the existing protection checks detect - # "source gone + destination exists" and return early. - with _post_process_locks_lock: - if context_key not in _post_process_locks: - _post_process_locks[context_key] = threading.Lock() - file_lock = _post_process_locks[context_key] - - file_lock.acquire() - try: - import os - import shutil - import time - from pathlib import Path - - # --- RACE CONDITION GUARD --- - # If source file is already gone, another thread (stream processor or - # verification worker) already processed it. Check immediately after - # acquiring the lock to avoid wasting time on stability checks / acoustid. - if not os.path.exists(file_path): - existing_final = context.get('_final_processed_path') - if existing_final and os.path.exists(existing_final): - logger.info(f"[Race Guard] Source gone but destination exists — already processed by another thread: {os.path.basename(existing_final)}") - return - logger.error(f"[Race Guard] Source file gone and no known destination — marking as failed: {os.path.basename(file_path)}") - context['_race_guard_failed'] = True - return - # --- END RACE CONDITION GUARD --- - - # --- FILE STABILITY CHECK --- - # Wait for the file to stop growing before processing. slskd may still be - # flushing write buffers when it reports "Completed". We poll the file size - # a few times; once it stabilises we know the write is finished. - _basename = os.path.basename(file_path) - _prev_size = -1 - for _stability_check in range(5): - try: - _cur_size = os.path.getsize(file_path) - except OSError: - _cur_size = -1 - if _cur_size == _prev_size and _cur_size > 0: - # Size unchanged — file is stable - break - _prev_size = _cur_size - if _stability_check == 0: - logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)") - time.sleep(1.5) - else: - logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)") - # --- END FILE STABILITY CHECK --- - - # --- ACOUSTID VERIFICATION --- - # Optional verification that downloaded audio matches expected track. - # Only runs if enabled and configured. Fails gracefully (skips on any error). - # Runs for ALL download sources — streaming APIs can return wrong versions - # (live, remix, cover) despite downloading by track ID. - _skip_acoustid = False - - try: - from core.acoustid_verification import AcoustIDVerification, VerificationResult - - verifier = AcoustIDVerification() - available, available_reason = verifier.quick_check_available() - - if available and not _skip_acoustid: - # Extract expected track info from context - track_info = context.get('track_info', {}) - original_search = context.get('original_search_result', {}) - spotify_artist = context.get('spotify_artist', {}) - - expected_track = ( - original_search.get('spotify_clean_title') or - track_info.get('name') or - original_search.get('title', '') - ) - - # Use track-level artist for verification, NOT album artist. - # For compilations, spotify_artist is "Various Artists" which - # will never match AcoustID's actual track artist. - expected_artist = '' - track_artists = track_info.get('artists', []) - if track_artists: - first = track_artists[0] - if isinstance(first, dict): - expected_artist = first.get('name', '') - elif isinstance(first, str): - expected_artist = first - # Fallback to album artist if no track artists available - if not expected_artist: - expected_artist = spotify_artist.get('name', '') - - if expected_track and expected_artist: - logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'") - verification_result, verification_msg = verifier.verify_audio_file( - file_path, - expected_track, - expected_artist, - context - ) - logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}") - context['_acoustid_result'] = verification_result.value - - if verification_result == VerificationResult.FAIL: - # Move to quarantine instead of Transfer - try: - quarantine_path = _move_to_quarantine(file_path, context, verification_msg) - logger.error(f"File quarantined due to verification failure: {quarantine_path}") - except Exception as quarantine_error: - # Quarantine failed — delete the known-wrong file instead - # NEVER save a file we've confirmed is wrong - logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}") - logger.error(f"Quarantine failed, deleting wrong file: {file_path}") - try: - os.remove(file_path) - except Exception as del_error: - logger.error(f"Could not delete wrong file either: {del_error}") - - # These always execute for FAIL — whether quarantine succeeded or not - context['_acoustid_quarantined'] = True - context['_acoustid_failure_msg'] = verification_msg - - # Clean up context - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Mark as failed in download tasks if we have task info - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {verification_msg}" - - # Call completion callback with failure - if task_id and batch_id: - _on_download_completed(batch_id, task_id, success=False) - - return # NEVER continue processing a known-wrong file - else: - logger.warning("AcoustID verification skipped: missing track/artist info") - context['_acoustid_result'] = 'skip' - else: - logger.info(f"ℹ️ AcoustID verification not available: {available_reason}") - context['_acoustid_result'] = 'disabled' - except Exception as verify_error: - # Any verification error should NOT block the download - fail open - logger.error(f"AcoustID verification error (continuing normally): {verify_error}") - context['_acoustid_result'] = 'error' - # --- END ACOUSTID VERIFICATION --- - - # --- SIMPLE DOWNLOAD HANDLING --- - # Check if this is a simple download (search page "Download ⬇" button only) - search_result = context.get('search_result', {}) - is_simple_download = search_result.get('is_simple_download', False) - - if is_simple_download: - # Simple transfer: move to Transfer folder, no metadata enhancement - logger.info(f"Processing simple download (no metadata enhancement): {file_path}") - - transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) - - # Check if this download has album info (from path or search result) - album_name = None - original_filename = search_result.get('filename', '') - - if '/' in original_filename or '\\' in original_filename: - # Get parent directory as album name - path_parts = original_filename.replace('\\', '/').split('/') - if len(path_parts) >= 2: - album_name = path_parts[-2] # Parent directory - - # If no album from path, check search result - if not album_name: - album_name = search_result.get('album') - - # Determine destination - filename = Path(file_path).name - - if album_name and album_name.lower() not in ['unknown', 'unknown album', '']: - # Has album info - create album folder - import re - album_name = re.sub(r'[<>:"/\\|?*]', '_', album_name).strip() - album_folder = Path(transfer_path) / album_name - album_folder.mkdir(parents=True, exist_ok=True) - destination = album_folder / filename - logger.info(f"Moving to album folder: {album_name}") - else: - # No album info - move directly to Transfer root (singles) - Path(transfer_path).mkdir(parents=True, exist_ok=True) - destination = Path(transfer_path) / filename - logger.info("Moving to Transfer root (single track)") - - _safe_move_file(file_path, destination) - logger.info(f"Moved simple download to: {destination}") - - # Clean up context - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Trigger library scan (using correct method name) - if web_scan_manager: - threading.Thread( - target=lambda: web_scan_manager.request_scan("Simple download completed"), - daemon=True - ).start() - - add_activity_item("", "Download Complete", f"{album_name}/{filename}", "Now") - logger.info(f"Simple download post-processing complete: {album_name}/{filename}") - - # Set flag in context so verification function knows this was fully handled - context['_simple_download_completed'] = True - context['_final_path'] = str(destination) - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - return - # --- END SIMPLE DOWNLOAD HANDLING --- - - logger.info(f"Starting robust post-processing for: {context_key}") - - spotify_artist = context.get("spotify_artist") - if not spotify_artist: - logger.error("Post-processing failed: Missing spotify_artist context.") - return - - # ── UNKNOWN ARTIST GUARD ── - # If artist name is junk, attempt to resolve from track metadata before proceeding. - # This prevents files from landing in "Unknown Artist/" folders. - _junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'} - _artist_name = (spotify_artist.get('name', '') if isinstance(spotify_artist, dict) else '').strip() - if _artist_name.lower() in _junk_artist_names: - logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve") - _resolved = False - track_info_guard = context.get("track_info", {}) or {} - original_search_guard = context.get("original_search_result", {}) or {} - - # Try 1: Pull artist from track_info.artists - _ti_artists = track_info_guard.get('artists', []) - if isinstance(_ti_artists, list) and _ti_artists: - _first = _ti_artists[0] - _name = _first.get('name', '') if isinstance(_first, dict) else str(_first) - if _name and _name.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _name.strip() - logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'") - _resolved = True - - # Try 2: Pull from original_search_result - if not _resolved: - _os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or '' - if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _os_artist.strip() - logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'") - _resolved = True - - # Try 3: Re-fetch from metadata source using track ID - if not _resolved: - _track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or '' - if _track_id: - try: - _fb_client = _get_metadata_fallback_client() - if hasattr(_fb_client, 'get_track_details'): - _details = _fb_client.get_track_details(str(_track_id)) - if _details and isinstance(_details, dict): - _d_artists = _details.get('artists', []) - if isinstance(_d_artists, list) and _d_artists: - _d_first = _d_artists[0] - _d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first) - if _d_name and _d_name.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _d_name.strip() - logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'") - _resolved = True - except Exception as _guard_err: - logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}") - - if not _resolved: - logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'") - - context['spotify_artist'] = spotify_artist - # ── END UNKNOWN ARTIST GUARD ── - - # Check if playlist folder mode is enabled (sync page playlists only) - track_info = context.get("track_info", {}) - playlist_folder_mode = track_info.get("_playlist_folder_mode", False) - - logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}") - logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}") - if track_info: - logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}") - - if playlist_folder_mode: - # Use shared path builder for playlist mode - playlist_name = track_info.get("_playlist_name", "Unknown Playlist") - logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}") - - file_ext = os.path.splitext(file_path)[1] - - # Build final path FIRST so we can check for already-processed files - final_path, _ = _build_final_path_for_track(context, spotify_artist, None, file_ext) - logger.info(f"Playlist mode final path: '{final_path}'") - - # RACE CONDITION GUARD: If source file is gone but destination exists, - # another thread (stream processor or verification worker) already moved it. - # Return early to avoid deleting the successfully processed file. - if not os.path.exists(file_path): - if os.path.exists(final_path): - logger.info(f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: {os.path.basename(final_path)}") - context['_final_processed_path'] = final_path - return - else: - pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}") - raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}") - - context['_audio_quality'] = _get_audio_quality_string(file_path) - if context['_audio_quality']: - logger.info(f"Audio quality detected: {context['_audio_quality']}") - - # FLAC bit depth filter - if _check_flac_bit_depth(file_path, context, context_key): - return - - # Enhance metadata before moving - try: - logger.warning(f"[Metadata Input] Playlist mode - artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") - _enhance_file_metadata(file_path, context, spotify_artist, None) - except Exception as meta_err: - import traceback - pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") - _wipe_source_tags(file_path) - - # Move file to playlist folder - logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") - _safe_move_file(file_path, final_path) - - # Store final path for verification wrapper (before conversions may override) - context['_final_processed_path'] = final_path - - # ReplayGain analysis — write track-level gain tags if enabled - if config_manager.get('post_processing.replaygain_enabled', False): - try: - from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF - if _rg_ffmpeg_ok(): - lufs, peak_dbfs = _rg_analyze(final_path) - gain_db = _RG_REF - lufs - _rg_write(final_path, gain_db, peak_dbfs) - pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}") - except Exception as rg_err: - pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") - - # Downsample hi-res FLAC to CD quality if enabled (must run before lossy copy) - downsampled_path = _downsample_hires_flac(final_path, context) - if downsampled_path: - final_path = downsampled_path - context['_final_processed_path'] = final_path - - # Lossy copy: create MP3 version if enabled - blasphemy_path = _create_lossy_copy(final_path) - if blasphemy_path: - context['_final_processed_path'] = blasphemy_path - - # Clean up empty directories in downloads folder - downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - _cleanup_empty_directories(downloads_path, file_path) - - logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}") - - # WISHLIST REMOVAL: Check if this track should be removed from wishlist - try: - _check_and_remove_from_wishlist(context) - except Exception as wishlist_error: - logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}") - - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - - # Mark as stream processed so the verification worker doesn't search - # for the file by its original Soulseek name (which no longer exists after rename) - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id and batch_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['stream_processed'] = True - download_tasks[task_id]['status'] = 'completed' - logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed") - _on_download_completed(batch_id, task_id, success=True) - - return # Skip normal album/artist folder structure processing - - is_album_download = context.get("is_album_download", False) - has_clean_spotify_data = context.get("has_clean_spotify_data", False) - - if is_album_download and has_clean_spotify_data: - # Build album_info directly from clean Spotify metadata (GUI PARITY) - logger.info("Album context with clean Spotify data found - using direct album info") - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album", {}) - - # Use clean Spotify metadata (matches GUI's SpotifyBasedSearchResult approach) - clean_track_name = original_search.get('spotify_clean_title', 'Unknown Track') - clean_album_name = original_search.get('spotify_clean_album', 'Unknown Album') - - logger.debug( - "Path 1 - Clean Spotify data path: keys=%s has_track_number=%s track_number=%s", - list(original_search.keys()), - 'track_number' in original_search, - original_search.get('track_number', 'NOT_FOUND'), - ) - - album_info = { - 'is_album': True, - 'album_name': clean_album_name, # Use clean Spotify album name - 'track_number': original_search.get('track_number', 1), - 'disc_number': original_search.get('disc_number', 1), - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('image_url'), - 'confidence': 1.0, # High confidence since we have clean Spotify data - 'source': 'clean_spotify_metadata' - } - - logger.info(f"Using clean Spotify album: '{clean_album_name}' for track: '{clean_track_name}'") - elif is_album_download: - # CRITICAL FIX: Album context without clean Spotify data - still force album treatment - logger.warning("Album context found but no clean Spotify data - using enhanced fallback") - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album", {}) - clean_track_name = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown Track') - - logger.debug( - "Path 2 - Enhanced fallback album context path: keys=%s has_track_number=%s track_number=%s spotify_album=%s", - list(original_search.keys()), - 'track_number' in original_search, - original_search.get('track_number', 'NOT_FOUND'), - spotify_album.get('name', 'NOT_FOUND'), - ) - - # ENHANCEMENT: Use spotify_clean_album if available for consistency - album_name = (original_search.get('spotify_clean_album') or - spotify_album.get('name') or - 'Unknown Album') - - album_info = { - 'is_album': True, # FORCE TRUE - user explicitly selected album for download - 'album_name': album_name, - 'track_number': original_search.get('track_number', 1), - 'disc_number': original_search.get('disc_number', 1), - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('image_url'), - 'confidence': 0.9, # Higher confidence - user explicitly chose album - 'source': 'enhanced_fallback_album_context' - } - logger.info(f"[FORCED ALBUM] Using album: '{album_name}' for track: '{clean_track_name}'") - else: - # For singles, we still need to detect if they belong to an album. - logger.info("Single track download - attempting album detection") - album_info = _detect_album_info_web(context, spotify_artist) - - # --- Album grouping resolution --- - # Only run smart grouping for singles/auto-detected albums. - # Explicit album downloads already have the correct Spotify album name — - # re-grouping would mangle names like "(Reworked and Remastered)" into "(Deluxe Edition)". - if album_info and album_info['is_album'] and not is_album_download: - logger.info( - "SMART ALBUM GROUPING for track=%r original_album=%r", - album_info.get('clean_track_name', 'Unknown'), - album_info.get('album_name', 'None'), - ) - - # Get original album name from context if available - original_album = None - if context.get("original_search_result", {}).get("album"): - original_album = context["original_search_result"]["album"] - - # Use the GUI's smart album grouping algorithm - consistent_album_name = _resolve_album_group(spotify_artist, album_info, original_album) - album_info['album_name'] = consistent_album_name - - logger.info("Album grouping complete: final_album=%r", consistent_album_name) - elif album_info and album_info['is_album'] and is_album_download: - logger.info( - "EXPLICIT ALBUM DOWNLOAD - preserving Spotify album name=%r; skipping smart grouping", - album_info.get('album_name', 'None'), - ) - - # 1. Get transfer path (directory creation handled by _build_final_path_for_track) - file_ext = os.path.splitext(file_path)[1] - context['_audio_quality'] = _get_audio_quality_string(file_path) - if context['_audio_quality']: - logger.info(f"Audio quality detected: {context['_audio_quality']}") - - # FLAC bit depth filter - if _check_flac_bit_depth(file_path, context, context_key): - return - - # 2. Build the final path using GUI-style track naming with multiple fallback sources - if album_info and album_info['is_album']: - album_name_sanitized = _sanitize_filename(album_info['album_name']) - - # --- GUI PARITY: Use multiple sources for clean track name --- - original_search = context.get("original_search_result", {}) - clean_track_name = album_info['clean_track_name'] - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - clean_track_name = original_search['spotify_clean_title'] - logger.info(f"Using Spotify clean title: '{clean_track_name}'") - # Priority 2: Album info clean name - elif album_info.get('clean_track_name'): - clean_track_name = album_info['clean_track_name'] - logger.info(f"Using album info clean name: '{clean_track_name}'") - # Priority 3: Original title as fallback - else: - clean_track_name = original_search.get('title', 'Unknown Track') - logger.warning(f"Using original title as fallback: '{clean_track_name}'") - - final_track_name_sanitized = _sanitize_filename(clean_track_name) - track_number = album_info['track_number'] - - logger.debug( - "Final track_number processing: source=%s album_info_track_number=%s track_number=%s", - album_info.get('source', 'unknown'), - album_info.get('track_number', 'NOT_FOUND'), - track_number, - ) - - # Fix: Handle None track_number - if track_number is None: - track_number = _extract_track_number_from_filename(file_path) - logger.info( - "Track number was None; extracted from filename=%r -> %s", - os.path.basename(file_path), - track_number, - ) - - # Ensure track_number is valid - if not isinstance(track_number, int) or track_number < 1: - logger.error(f"Invalid track number ({track_number}), defaulting to 1") - track_number = 1 - - logger.debug(f"FINAL track_number used for filename: {track_number}") - - # CRITICAL FIX: Update album_info with corrected track_number for metadata enhancement - album_info['track_number'] = track_number - album_info['clean_track_name'] = clean_track_name # Ensure clean name is in album_info - logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata") - - # Use shared path builder for album mode - final_path, _ = _build_final_path_for_track(context, spotify_artist, album_info, file_ext) - logger.info(f"Album path: '{final_path}'") - else: - # Single track structure: Transfer/ARTIST/ARTIST - SINGLE/SINGLE.ext - # --- GUI PARITY: Use multiple sources for clean track name --- - original_search = context.get("original_search_result", {}) - clean_track_name = album_info['clean_track_name'] - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - clean_track_name = original_search['spotify_clean_title'] - logger.info(f"Using Spotify clean title: '{clean_track_name}'") - # Priority 2: Album info clean name - elif album_info and album_info.get('clean_track_name'): - clean_track_name = album_info['clean_track_name'] - logger.info(f"Using album info clean name: '{clean_track_name}'") - # Priority 3: Original title as fallback - else: - clean_track_name = original_search.get('title', 'Unknown Track') - logger.warning(f"Using original title as fallback: '{clean_track_name}'") - - # Ensure clean name is in album_info for path builder - if album_info: - album_info['clean_track_name'] = clean_track_name - - # Use shared path builder for single mode - final_path, _ = _build_final_path_for_track(context, spotify_artist, album_info, file_ext) - logger.info(f"Single path: '{final_path}'") - - # Store the actual computed path so verification uses this exact path - # instead of recomputing independently (which can produce mismatches) - context['_final_processed_path'] = final_path - - # 3. Enhance metadata, move file, download art, and cleanup - try: - logger.warning(f"[Metadata Input] artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") - if album_info: - logger.warning(f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, source: {album_info.get('source', 'unknown')}") - else: - logger.info("[Metadata Input] album_info: None (single track)") - _enhance_file_metadata(file_path, context, spotify_artist, album_info) - except Exception as meta_err: - import traceback - pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") - # Wipe source tags even though enhancement failed — prevents Soulseek - # uploader's MusicBrainz IDs from causing album splits in Navidrome - _wipe_source_tags(file_path) - - # Detect enhance mode from track context - _enhance_source_info = context.get('track_info', {}).get('source_info') or {} - if isinstance(_enhance_source_info, str): - try: - _enhance_source_info = json.loads(_enhance_source_info) - except (json.JSONDecodeError, TypeError): - _enhance_source_info = {} - is_enhance_download = _enhance_source_info.get('enhance', False) - - logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") - if os.path.exists(final_path): - # PROTECTION: If destination already exists, check before overwriting - # If the source file is gone, another thread already handled this - don't delete the destination - if not os.path.exists(file_path): - logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}") - return - try: - from mutagen import File as MutagenFile - existing_file = MutagenFile(final_path) - has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2 # More than basic tags - - if has_metadata and not is_enhance_download: - # Check if incoming file is higher quality and setting is enabled - _replace_lower = config_manager.get('import.replace_lower_quality', False) - if _replace_lower: - _existing_tier = _get_quality_tier_from_extension(final_path) - _incoming_tier = _get_quality_tier_from_extension(file_path) - if _incoming_tier[1] < _existing_tier[1]: - # Incoming is higher quality (lower tier number) — replace - logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except Exception as e: - logger.error(f"[Quality Replace] Could not remove existing file: {e}") - else: - logger.info(f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: {os.path.basename(final_path)}") - try: - os.remove(file_path) - except FileNotFoundError: - pass - except Exception as e: - logger.error(f"[Protection] Error removing redundant file: {e}") - return - else: - logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}") - logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}") - try: - os.remove(file_path) - except FileNotFoundError: - logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}") - except Exception as e: - logger.error(f"[Protection] Error removing redundant file: {e}") - return # Don't overwrite the good file - elif is_enhance_download: - # ENHANCE BYPASS: Allow overwrite — backup original, then remove to allow move - logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except Exception as e: - logger.error(f"[Enhance] Could not remove existing file for replacement: {e}") - else: - logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except FileNotFoundError: - pass # It was just there, but now gone? - except Exception as check_error: - logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}") - try: - if os.path.exists(final_path): - os.remove(final_path) - except Exception as e: - logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}") - - # --- PRE-MOVE SOURCE CHECK --- - # Right before moving, verify the source file still exists. - # Another thread (Stream Processor or Verification Worker) may have - # already moved this file during the sleep + metadata enhancement window. - if not os.path.exists(file_path): - if os.path.exists(final_path): - logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}") - # Still do cover art + lyrics since the other thread might not have finished those - _download_cover_art(album_info, os.path.dirname(final_path), context) - _generate_lrc_file(final_path, context, spotify_artist, album_info) - return - else: - # Source gone, exact destination not found — check if stream processor already - # moved it with a quality tag (e.g. "track [FLAC 24bit].flac" vs "track.flac") - expected_dir = os.path.dirname(final_path) - expected_stem = os.path.splitext(os.path.basename(final_path))[0] - expected_ext = os.path.splitext(final_path)[1] - found_variant = None - # Also check for lossy output if Blasphemy Mode may have deleted the .flac - check_exts = {expected_ext} - if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False): - _lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'} - _lossy_codec = config_manager.get('lossy_copy.codec', 'mp3') - check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3')) - if os.path.exists(expected_dir): - for f in os.listdir(expected_dir): - # Match files that start with the expected stem and have a matching extension - # This catches "01 - track [FLAC 24bit].flac" when expecting "01 - track.flac" - # and "01 - track [MP3-320].mp3" when Blasphemy Mode deleted the FLAC - f_ext = os.path.splitext(f)[1].lower() - if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem): - found_variant = os.path.join(expected_dir, f) - break - if found_variant: - logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}") - context['_final_processed_path'] = found_variant - _download_cover_art(album_info, expected_dir, context) - _generate_lrc_file(found_variant, context, spotify_artist, album_info) - return - else: - logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}") - raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}") - - _safe_move_file(file_path, final_path) - - # ENHANCE CLEANUP: If format changed (e.g. MP3→FLAC), remove old-format file - if is_enhance_download and _enhance_source_info.get('original_file_path'): - original_enhance_path = _enhance_source_info['original_file_path'] - if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path): - try: - os.remove(original_enhance_path) - old_fmt = os.path.splitext(original_enhance_path)[1] - new_fmt = os.path.splitext(final_path)[1] - logger.info(f"[Enhance] Upgraded {old_fmt} → {new_fmt}: {os.path.basename(final_path)}") - except Exception as e: - logger.error(f"[Enhance] Could not remove old-format file: {e}") - elif is_enhance_download: - old_fmt = _enhance_source_info.get('original_format', 'unknown') - new_fmt = os.path.splitext(final_path)[1] - logger.info(f"[Enhance] Replaced in-place ({old_fmt} → {new_fmt}): {os.path.basename(final_path)}") - - _download_cover_art(album_info, os.path.dirname(final_path), context) - - # 4. Generate LRC lyrics file at final location (elegant addition) - _generate_lrc_file(final_path, context, spotify_artist, album_info) - - # 5. ReplayGain analysis — write track-level gain tags if enabled - if config_manager.get('post_processing.replaygain_enabled', False): - try: - from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF - if _rg_ffmpeg_ok(): - lufs, peak_dbfs = _rg_analyze(final_path) - gain_db = _RG_REF - lufs - _rg_write(final_path, gain_db, peak_dbfs) - pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}") - except Exception as rg_err: - pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") - - # Downsample hi-res FLAC to CD quality if enabled (must run before lossy copy) - downsampled_path = _downsample_hires_flac(final_path, context) - if downsampled_path: - final_path = downsampled_path - context['_final_processed_path'] = final_path - - # Lossy copy: create MP3 version if enabled - blasphemy_path = _create_lossy_copy(final_path) - if blasphemy_path: - context['_final_processed_path'] = blasphemy_path - - downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - _cleanup_empty_directories(downloads_path, file_path) - - logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}") - - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - _record_soulsync_library_entry(context, spotify_artist, album_info) - - # RETAG DATA CAPTURE: Record completed album/single downloads for retag tool - try: - if not playlist_folder_mode: - completed_path = context.get('_final_processed_path', final_path) - _record_retag_download(context, spotify_artist, album_info, completed_path) - except Exception as retag_err: - logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}") - - # REPAIR: Register album folder for repair scanning when batch completes - try: - completed_path = context.get('_final_processed_path', final_path) - batch_id_for_repair = context.get('batch_id') - if completed_path and batch_id_for_repair and repair_worker: - album_folder = os.path.dirname(str(completed_path)) - if album_folder: - repair_worker.register_folder(batch_id_for_repair, album_folder) - except Exception as repair_err: - logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}") - - # ALBUM CONSISTENCY: Register completed file for post-batch MB tag reconciliation - try: - completed_path = context.get('_final_processed_path', final_path) - batch_id_for_consistency = context.get('batch_id') - if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'): - _original_search = context.get('original_search_result', {}) - _file_info = { - 'path': str(completed_path), - 'track_number': album_info.get('track_number', 1), - 'disc_number': album_info.get('disc_number', 1), - 'title': _original_search.get('spotify_clean_title', '') or album_info.get('clean_track_name', ''), - } - with tasks_lock: - if batch_id_for_consistency in download_batches: - download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info) - except Exception as cons_err: - logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}") - - # WISHLIST REMOVAL: Check if this track should be removed from wishlist after successful download - try: - _check_and_remove_from_wishlist(context) - except Exception as wishlist_error: - logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}") - - # Call completion callback for missing downloads tasks to start next batch - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id and batch_id: - logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}") - - # Mark task as stream processed and set terminal status so - # _validate_worker_counts won't count this task as active - # (prevents active_count inflation race). - # NOTE: Only set status here — don't call _mark_task_completed() because - # _run_post_processing_worker will call it later with the session counter - # increment. Calling it here too would double-count the download. - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['stream_processed'] = True - download_tasks[task_id]['status'] = 'completed' - logger.info(f"[Post-Process] Marked task {task_id} as completed") - - _on_download_completed(batch_id, task_id, success=True) - - except Exception as e: - import traceback - pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}") - pp_logger.info(traceback.format_exc()) - logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}") - traceback.print_exc() - - # Only retry if the source file still exists - otherwise retrying is pointless - # and creates an infinite loop of failures - import os as _os - source_exists = _os.path.exists(file_path) if file_path else False - if source_exists: - # Remove from processed set so it can be retried - if context_key in _processed_download_ids: - _processed_download_ids.remove(context_key) - logger.warning(f"Removed {context_key} from processed set - will retry on next check") - - # Re-add to matched context for retry - with matched_context_lock: - if context_key not in matched_downloads_context: - matched_downloads_context[context_key] = context - logger.warning(f"Re-added {context_key} to context for retry") - else: - logger.warning(f"Source file gone, not retrying: {context_key}") - finally: - file_lock.release() - # Clean up the lock entry to prevent unbounded memory growth - with _post_process_locks_lock: - _post_process_locks.pop(context_key, None) + from core.import_pipeline import post_process_matched_download + return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime()) # Keep track of processed downloads to avoid re-processing _processed_download_ids = set() @@ -22236,12 +18330,6 @@ _processed_download_ids = set() # so we only log the warning once per key instead of spamming every poll cycle _stale_transfer_keys = set() -# Per-context-key locks to prevent two threads from processing the same file simultaneously. -# Without this, the Stream Processor and Verification Worker race to move the same source file, -# and the loser gets a FileNotFoundError because the winner already moved it. -_post_process_locks = {} # {context_key: threading.Lock()} -_post_process_locks_lock = threading.Lock() # protects the dict itself - # --- File Discovery Retry System --- # Prevents race condition where slskd reports completion before file is written to disk # Tracks retry attempts per download to give files time to finish writing @@ -22249,83 +18337,6 @@ _download_retry_attempts = {} # {context_key: {'count': N, 'first_attempt': tim _download_retry_max = 10 # Max retries before giving up (10 seconds with 1s poll interval) _download_retry_lock = threading.Lock() -def _record_retag_download(context, spotify_artist, album_info, final_path): - """Record a completed download in the retag tables for later re-tagging.""" - from database.music_database import get_database - db = get_database() - - # Extract artist name - if isinstance(spotify_artist, dict): - artist_name = spotify_artist.get('name', 'Unknown Artist') - else: - artist_name = getattr(spotify_artist, 'name', 'Unknown Artist') - - spotify_album = context.get('spotify_album', {}) - original_search = context.get('original_search_result', {}) - track_info = context.get('track_info', {}) - - is_album = album_info and album_info.get('is_album', False) - group_type = 'album' if is_album else 'single' - album_name = album_info.get('album_name', '') if album_info else ( - original_search.get('spotify_clean_title', 'Unknown')) - - # Determine album IDs (Spotify vs iTunes) — skip beatport_ prefixed IDs - spotify_album_id = None - itunes_album_id = None - if spotify_album: - album_id_raw = str(spotify_album.get('id', '')) - if album_id_raw and album_id_raw.isdigit(): - itunes_album_id = album_id_raw - elif album_id_raw and not album_id_raw.startswith('beatport_'): - spotify_album_id = album_id_raw - - image_url = album_info.get('album_image_url') if album_info else None - total_tracks = spotify_album.get('total_tracks', 1) if spotify_album else 1 - release_date = spotify_album.get('release_date', '') if spotify_album else '' - - # Find or create group (avoid duplicating for multi-track albums) - group_id = db.find_retag_group(artist_name, album_name) - if group_id is None: - group_id = db.add_retag_group( - group_type=group_type, artist_name=artist_name, album_name=album_name, - image_url=image_url, spotify_album_id=spotify_album_id, - itunes_album_id=itunes_album_id, total_tracks=total_tracks, - release_date=release_date - ) - if group_id is None: - return - - # Track details - track_number = album_info.get('track_number', 1) if album_info else 1 - disc_number = original_search.get('disc_number') or ( - album_info.get('disc_number', 1) if album_info else 1) - title = original_search.get('spotify_clean_title') or ( - album_info.get('clean_track_name', 'Unknown Track') if album_info else 'Unknown Track') - file_format = os.path.splitext(str(final_path))[1].lstrip('.').lower() - - # Track IDs (Spotify vs iTunes) — skip beatport_ prefixed IDs - spotify_track_id = None - itunes_track_id = None - if track_info and track_info.get('id'): - tid = str(track_info['id']) - if tid.isdigit(): - itunes_track_id = tid - elif not tid.startswith('beatport_'): - spotify_track_id = tid - - # Avoid duplicate track entries - if not db.retag_track_exists(group_id, str(final_path)): - db.add_retag_track( - group_id=group_id, track_number=track_number, disc_number=disc_number, - title=title, file_path=str(final_path), file_format=file_format, - spotify_track_id=spotify_track_id, itunes_track_id=itunes_track_id - ) - logger.info(f"[Retag] Recorded track for retag: '{title}' in '{album_name}'") - - # Cap retag groups at 100, remove oldest - db.trim_retag_groups(100) - - def _execute_retag(group_id, album_id): """Execute a retag operation: re-tag files in a group with metadata from a new album match.""" global retag_state @@ -27449,7 +23460,7 @@ def _on_download_completed(batch_id, task_id, success=True): artist_name=_cons_artist_name, mb_service=_cons_mb_svc, total_discs=_cons_album.get('total_discs', 1), - file_lock_fn=_get_file_lock, + file_lock_fn=get_file_lock, ) if _cons_result.get('success'): logger.info(f"[Album Consistency] {_cons_result['tags_written']}/{_cons_result['total_files']} files " @@ -27533,7 +23544,7 @@ def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json): if release and release.get('id'): release_mbid = release['id'] _artist_key = artist_name_pf.lower().strip() - _rc_key_norm = (_normalize_album_cache_key(album_name_pf), _artist_key) + _rc_key_norm = (normalize_album_cache_key(album_name_pf), _artist_key) _rc_key_exact = (album_name_pf.lower().strip(), _artist_key) with _mb_release_cache_lock: _mb_release_cache[_rc_key_norm] = release_mbid @@ -28123,19 +24134,19 @@ def _run_post_processing_worker(task_id, batch_id): original_search = context.get("original_search_result", {}) logger.info(f"[Post-Processing] original_search keys: {list(original_search.keys())}") - spotify_clean_title = original_search.get('spotify_clean_title') + clean_title = get_import_clean_title(context, default=original_search.get('title', '')) track_number = original_search.get('track_number') - logger.info(f"[Post-Processing] spotify_clean_title: '{spotify_clean_title}', track_number: {track_number}") + logger.info(f"[Post-Processing] clean_title: '{clean_title}', track_number: {track_number}") - if spotify_clean_title and track_number: + if clean_title and track_number: # Generate expected final filename that stream processor would create # Pattern: f"{track_number:02d} - {clean_title}.flac" - sanitized_title = spotify_clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') + sanitized_title = clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') expected_final_filename = f"{track_number:02d} - {sanitized_title}.flac" logger.info(f"[Post-Processing] Generated expected final filename: {expected_final_filename}") else: - logger.warning(f"[Post-Processing] Missing required data - spotify_clean_title: {bool(spotify_clean_title)}, track_number: {bool(track_number)}") + logger.warning(f"[Post-Processing] Missing required data - clean_title: {bool(clean_title)}, track_number: {bool(track_number)}") except Exception as e: logger.error(f"[Post-Processing] Error generating expected filename: {e}") import traceback @@ -28160,19 +24171,19 @@ def _run_post_processing_worker(task_id, batch_id): original_search = context.get("original_search_result", {}) logger.info(f"[Post-Processing] fuzzy context original_search keys: {list(original_search.keys())}") - spotify_clean_title = original_search.get('spotify_clean_title') + clean_title = get_import_clean_title(context, default=original_search.get('title', '')) track_number = original_search.get('track_number') - logger.info(f"[Post-Processing] fuzzy context spotify_clean_title: '{spotify_clean_title}', track_number: {track_number}") + logger.info(f"[Post-Processing] fuzzy context clean_title: '{clean_title}', track_number: {track_number}") - if spotify_clean_title and track_number: + if clean_title and track_number: # Generate expected final filename that stream processor would create # Pattern: f"{track_number:02d} - {clean_title}.flac" - sanitized_title = spotify_clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') + sanitized_title = clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') expected_final_filename = f"{track_number:02d} - {sanitized_title}.flac" logger.info(f"[Post-Processing] Generated expected final filename from fuzzy match: {expected_final_filename}") else: - logger.warning(f"[Post-Processing] Missing required data from fuzzy match - spotify_clean_title: {bool(spotify_clean_title)}, track_number: {bool(track_number)}") + logger.warning(f"[Post-Processing] Missing required data from fuzzy match - clean_title: {bool(clean_title)}, track_number: {bool(track_number)}") except Exception as e: logger.error(f"[Post-Processing] Error generating expected filename from fuzzy match: {e}") import traceback @@ -28312,22 +24323,22 @@ def _run_post_processing_worker(task_id, batch_id): # Attempt to complete metadata enhancement using context if context and expected_final_filename: try: + context = normalize_import_context(context) # Extract required data from context - original_search = context.get("original_search_result", {}) - spotify_artist = context.get("spotify_artist") - spotify_album = context.get("spotify_album") + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + album_context = get_import_context_album(context) - if spotify_artist and spotify_album: + if artist_context and album_context: # CRITICAL FIX: Create album_info dict with proper structure for metadata enhancement # This must match the format used in main stream processor to ensure consistency # Extract track number from context (should be available from fuzzy match) - original_search = context.get("original_search_result", {}) track_number = original_search.get('track_number', 1) # If no track number in context, extract from filename if track_number == 1 and found_file: - track_number = _extract_track_number_from_filename(found_file) + track_number = extract_track_number_from_filename(found_file) logger.warning( "[Verification] missing track_number; extracted from filename=%r -> %s", os.path.basename(found_file), @@ -28339,17 +24350,22 @@ def _run_post_processing_worker(task_id, batch_id): logger.error(f"[Verification] Invalid track number ({track_number}), defaulting to 1") track_number = 1 - # Get clean track name - clean_track_name = (original_search.get('spotify_clean_title') or - original_search.get('title', 'Unknown Track')) + # Get clean track name + clean_track_name = get_import_clean_title(context, default=original_search.get('title', 'Unknown Track')) + album_name = get_import_clean_album(context, default=album_context.get('name', 'Unknown Album')) + album_image_url = album_context.get('image_url') + if not album_image_url and album_context.get('images'): + album_images = album_context.get('images', []) + if album_images and isinstance(album_images[0], dict): + album_image_url = album_images[0].get('url') album_info = { 'is_album': True, # CRITICAL: Mark as album track - 'album_name': spotify_album.get('name', 'Unknown Album'), # CORRECT KEY + 'album_name': album_name, 'track_number': track_number, # CORRECTED TRACK NUMBER 'disc_number': original_search.get('disc_number', 1), 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('images', [{}])[0].get('url') if spotify_album.get('images') else None, + 'album_image_url': album_image_url, 'confidence': 0.9, 'source': 'verification_worker_corrected' } @@ -28366,19 +24382,19 @@ def _run_post_processing_worker(task_id, batch_id): original_album_ctx = raw_album_ctx['name'] else: original_album_ctx = None - consistent_album_name = _resolve_album_group(spotify_artist, album_info, original_album_ctx) + consistent_album_name = _resolve_album_group(artist_context, album_info, original_album_ctx) album_info['album_name'] = consistent_album_name except Exception as group_err: logger.error(f"[Verification] Album grouping failed, using raw name: {group_err}") else: - logger.info(f"[Verification] Explicit album download - preserving Spotify album name: '{album_info['album_name']}'") + logger.info(f"[Verification] Explicit album download - preserving album name: '{album_info['album_name']}'") logger.info(f"[Verification] Created proper album_info - track_number: {track_number}, album: {album_info['album_name']}") logger.info(f"[Post-Processing] Attempting metadata enhancement for: {found_file}") - logger.warning(f"[Metadata Input] Verification worker - artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") + logger.warning(f"[Metadata Input] Verification worker - artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})") logger.warning(f"[Metadata Input] Verification worker - album: '{album_info.get('album_name', 'MISSING')}', track#: {album_info.get('track_number', 'MISSING')}, source: {album_info.get('source', 'unknown')}") - enhancement_success = _enhance_file_metadata(found_file, context, spotify_artist, album_info) + enhancement_success = _enhance_file_metadata(found_file, context, artist_context, album_info) if enhancement_success: with tasks_lock: @@ -28388,8 +24404,8 @@ def _run_post_processing_worker(task_id, batch_id): else: logger.info(f"[Post-Processing] Metadata enhancement returned False for: {os.path.basename(found_file)}") else: - logger.warning("[Post-Processing] Missing spotify_artist or spotify_album in context") - logger.info(f"[Post-Processing] spotify_artist: {spotify_artist is not None}, spotify_album: {spotify_album is not None}") + logger.warning("[Post-Processing] Missing artist or album in context") + logger.info(f"[Post-Processing] artist_context: {artist_context is not None}, album_context: {album_context is not None}") # Wipe source tags even without full enhancement — prevents # Soulseek uploader's MusicBrainz IDs from causing album splits if found_file and os.path.exists(found_file): @@ -28466,7 +24482,6 @@ def _run_post_processing_worker(task_id, batch_id): download_tasks[task_id]['error_message'] = f"Critical post-processing error: {str(e)}" _on_download_completed(batch_id, task_id, success=False) - def _download_track_worker(task_id, batch_id=None): """ Enhanced download worker that matches the GUI's exact retry logic. @@ -29126,7 +25141,7 @@ def _get_staging_file_cache(batch_id): if batch_id in _staging_cache: return _staging_cache[batch_id] - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): with _staging_cache_lock: _staging_cache[batch_id] = [] @@ -30654,7 +26669,7 @@ def _check_batch_completion_v2(batch_id): artist_name=_cons_artist_name, mb_service=_cons_mb_svc, total_discs=_cons_album.get('total_discs', 1), - file_lock_fn=_get_file_lock, + file_lock_fn=get_file_lock, ) if _cons_result.get('success'): logger.info(f"[Album Consistency V2] {_cons_result['tags_written']}/{_cons_result['total_files']} files " @@ -45808,142 +41823,6 @@ def cancel_listenbrainz_sync(playlist_mbid): logger.error(f"Error cancelling ListenBrainz sync: {e}") return jsonify({"error": str(e)}), 500 - -# OLD ENDPOINT - REMOVE ALL THE CODE BELOW FOR THE OLD IMPLEMENTATION -def _old_get_listenbrainz_playlist_tracks_DEPRECATED(playlist_mbid): - """DEPRECATED - Old implementation that fetches from API""" - try: - from core.listenbrainz_client import ListenBrainzClient - - client = ListenBrainzClient() - - playlist = client.get_playlist_details(playlist_mbid, fetch_metadata=True) - - if not playlist: - return jsonify({ - "success": False, - "error": "Playlist not found or not accessible" - }), 404 - - # Extract tracks from JSPF format - jspf_tracks = playlist.get('track', []) - - # Convert to our standard format - prepare tracks first without cover art - tracks = [] - logger.info(f"Processing {len(jspf_tracks)} tracks from playlist") - - # First pass: extract all track data without cover art - track_data_list = [] - for idx, track in enumerate(jspf_tracks): - # Get recording MBID from identifier - recording_mbid = None - identifiers = track.get('identifier', []) - for identifier in identifiers: - if 'musicbrainz.org/recording/' in identifier: - recording_mbid = identifier.split('/')[-1] - break - - # Get extension data (has MusicBrainz metadata) - extension = track.get('extension', {}) - mb_data = extension.get('https://musicbrainz.org/doc/jspf#track', {}) - - if idx == 0: - logger.debug(f"Sample track extension data: {extension}") - logger.debug(f"Sample mb_data keys: {mb_data.keys() if mb_data else 'None'}") - - # Extract release MBID for cover art - release_mbid = None - if mb_data: - # Check in additional_metadata first - additional_metadata = mb_data.get('additional_metadata', {}) - if 'caa_release_mbid' in additional_metadata: - release_mbid = additional_metadata['caa_release_mbid'] - # Fallback to top-level release_mbid - elif 'release_mbid' in mb_data: - release_mbid = mb_data['release_mbid'] - - if idx == 0: - logger.debug(f"🆔 First track release_mbid: {release_mbid}") - - track_data = { - 'track_name': track.get('title', 'Unknown Track'), - 'artist_name': track.get('creator', 'Unknown Artist'), - 'album_name': track.get('album', 'Unknown Album'), - 'duration_ms': track.get('duration', 0), - 'mbid': recording_mbid, - 'release_mbid': release_mbid, - 'album_cover_url': None, # Will be fetched in parallel - 'additional_metadata': mb_data - } - - track_data_list.append(track_data) - - # Second pass: fetch cover art in parallel using threading (much faster) - from concurrent.futures import ThreadPoolExecutor, as_completed - import time - - def fetch_cover_art(track_data): - """Fetch cover art for a single track""" - release_mbid = track_data.get('release_mbid') - if not release_mbid: - return None - - try: - cover_art_url = f"https://coverartarchive.org/release/{release_mbid}" - cover_response = requests.get(cover_art_url, timeout=3) - - if cover_response.status_code == 200: - cover_data = cover_response.json() - images = cover_data.get('images', []) - - # Get front cover - for img in images: - if img.get('front'): - return img.get('thumbnails', {}).get('small') or img.get('image') - - # Fallback to first image - if images: - return images[0].get('thumbnails', {}).get('small') or images[0].get('image') - except: - pass - - return None - - logger.info(f"Fetching cover art for {len(track_data_list)} tracks in parallel...") - start_time = time.time() - - # Fetch up to 10 covers at a time - with ThreadPoolExecutor(max_workers=10) as executor: - future_to_track = {executor.submit(fetch_cover_art, track): idx - for idx, track in enumerate(track_data_list)} - - for future in as_completed(future_to_track): - idx = future_to_track[future] - try: - cover_url = future.result() - if cover_url: - track_data_list[idx]['album_cover_url'] = cover_url - except Exception as e: - pass - - elapsed = time.time() - start_time - covers_found = sum(1 for t in track_data_list if t.get('album_cover_url')) - logger.info(f"Fetched {covers_found}/{len(track_data_list)} covers in {elapsed:.2f}s") - - tracks = track_data_list - - return jsonify({ - "success": True, - "tracks": tracks, - "track_count": len(tracks) - }) - - except Exception as e: - logger.error(f"Error getting ListenBrainz playlist tracks: {e}") - import traceback - traceback.print_exc() - return jsonify({"success": False, "error": str(e)}), 500 - @app.route('/api/metadata/start', methods=['POST']) def start_metadata_update(): """Start the metadata update process - EXACT copy of dashboard.py logic""" @@ -52006,17 +47885,11 @@ def repair_job_progress(): AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif', '.ape'} -def _get_staging_path(): - """Get the resolved staging folder path.""" - raw = config_manager.get('import.staging_path', './Staging') - return docker_resolve_path(raw) - - @app.route('/api/import/staging/files', methods=['GET']) def import_staging_files(): """Scan the staging folder and return audio files with tag metadata.""" try: - staging_path = _get_staging_path() + staging_path = get_staging_path() os.makedirs(staging_path, exist_ok=True) files = [] @@ -52028,7 +47901,7 @@ def import_staging_files(): full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) - meta = _read_staging_file_metadata(full_path, rel_path) + meta = read_staging_file_metadata(full_path, rel_path) files.append({ 'filename': fname, @@ -52058,7 +47931,7 @@ def import_staging_groups(): the same album+artist combo. Returns groups sorted by file count descending. """ try: - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): return jsonify({'success': True, 'groups': []}) @@ -52072,7 +47945,7 @@ def import_staging_groups(): full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) - meta = _read_staging_file_metadata(full_path, rel_path) + meta = read_staging_file_metadata(full_path, rel_path) album = meta['album'] artist = meta['albumartist'] or meta['artist'] if not album or not artist: @@ -52118,7 +47991,7 @@ def import_staging_groups(): def import_staging_hints(): """Extract album search hints from staging folder (tags + folder names). Fast — no Spotify calls.""" try: - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): return jsonify({'success': True, 'hints': []}) @@ -52182,34 +48055,20 @@ def import_staging_hints(): @app.route('/api/import/search/albums', methods=['GET']) def import_search_albums(): - """Search for albums via Spotify for import matching.""" + """Search for albums using the active metadata provider.""" try: query = request.args.get('q', '').strip() if not query: return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 limit = min(int(request.args.get('limit', 12)), 50) + from core.metadata_service import get_primary_source - if _is_hydrabase_active(): - albums = hydrabase_client.search_albums(query, limit=limit) - else: - if hydrabase_worker and dev_mode_enabled: - hydrabase_worker.enqueue(query, 'albums') - albums = spotify_client.search_albums(query, limit=limit) + if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: + hydrabase_worker.enqueue(query, 'albums') - results = [] - for a in albums: - results.append({ - 'id': a.id, - 'name': a.name, - 'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist', - 'release_date': a.release_date or '', - 'total_tracks': a.total_tracks, - 'image_url': a.image_url, - 'album_type': a.album_type or 'album' - }) - - return jsonify({'success': True, 'albums': results}) + albums = search_import_albums(query, limit=limit) + return jsonify({'success': True, 'albums': albums}) except Exception as e: logger.error(f"Error searching albums for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52219,204 +48078,24 @@ def import_search_albums(): def import_album_match(): """Match staging files to an album's tracklist.""" try: - data = request.get_json() + data = request.get_json() or {} album_id = data.get('album_id') album_name = data.get('album_name', '') album_artist = data.get('album_artist', '') + source = str(data.get('source') or '').strip().lower() # Optional: only match specific files (from auto-group selection) filter_file_paths = set(data.get('file_paths', [])) if not album_id: return jsonify({'success': False, 'error': 'Missing album_id'}), 400 - spotify_tracks = None - album_info = None - - # Try Hydrabase first when active — look up by album soul_id - if _is_hydrabase_active(): - try: - hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50) - if hydra_tracks: - spotify_tracks = [] - for t in hydra_tracks: - artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else [] - spotify_tracks.append({ - 'name': t.name, - 'track_number': t.track_number or 0, - 'disc_number': t.disc_number or 1, - 'duration_ms': t.duration_ms, - 'id': t.id, - 'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list], - 'uri': '' - }) - album_info = { - 'id': album_id, - 'name': album_name or hydra_tracks[0].album or '', - 'artist': album_artist, - 'artists': [album_artist] if album_artist else [], - 'release_date': '', - 'total_tracks': len(spotify_tracks), - 'image_url': None, - 'genres': [] - } - logger.info(f"Hydrabase album_tracks returned {len(spotify_tracks)} tracks for album_id '{album_id}'") - except Exception as e: - logger.warning(f"Hydrabase album_tracks failed, falling back to Spotify: {e}") - spotify_tracks = None - - # Fall back to Spotify - if spotify_tracks is None: - album_data = spotify_client.get_album(album_id) - if not album_data: - return jsonify({'success': False, 'error': 'Album not found'}), 404 - - tracks_data = spotify_client.get_album_tracks(album_id) - if not tracks_data or 'items' not in tracks_data: - return jsonify({'success': False, 'error': 'Could not get album tracks'}), 500 - - spotify_tracks = tracks_data['items'] - - # Build album summary - album_artists = [a['name'] for a in album_data.get('artists', [])] - album_info = { - 'id': album_id, - 'name': album_data.get('name', 'Unknown Album'), - 'artist': ', '.join(album_artists), - 'artists': album_artists, - 'release_date': album_data.get('release_date', ''), - 'total_tracks': album_data.get('total_tracks', len(spotify_tracks)), - 'image_url': (album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None), - 'genres': album_data.get('genres', []) - } - - # Get artist info for context building later - if album_data.get('artists'): - primary_artist = album_data['artists'][0] - album_info['artist_id'] = primary_artist.get('id', '') - - # Scan staging files - staging_path = _get_staging_path() - staging_files = [] - for root, _dirs, filenames in os.walk(staging_path): - for fname in filenames: - ext = os.path.splitext(fname)[1].lower() - if ext not in AUDIO_EXTENSIONS: - continue - full_path = os.path.join(root, fname) - - meta = _read_staging_file_metadata(full_path, fname) - - staging_files.append({ - 'filename': fname, - 'full_path': full_path, - 'title': meta['title'], - 'artist': meta['albumartist'] or meta['artist'], - 'album': meta['album'], - 'track_number': meta['track_number'], - 'disc_number': meta['disc_number'], - }) - - # Filter to specific files if requested (auto-group selection) - if filter_file_paths: - staging_files = [sf for sf in staging_files if sf['full_path'] in filter_file_paths] - - # Match each Spotify track to the best staging file - matches = [] - used_files = set() - album_name_for_match = album_info.get('name', '') - - for sp_track in spotify_tracks: - sp_name = sp_track.get('name', '') - sp_number = sp_track.get('track_number', 0) - sp_disc = sp_track.get('disc_number', 1) - sp_artists = sp_track.get('artists', []) - sp_artist_name = sp_artists[0]['name'] if sp_artists and isinstance(sp_artists[0], dict) else (sp_artists[0] if sp_artists else '') - - best_match = None - best_score = 0.0 - - for i, sf in enumerate(staging_files): - if i in used_files: - continue - - score = 0.0 - - # Title similarity (weight 0.45) - title_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sp_name), - matching_engine.normalize_string(sf['title'] or '') - ) - score += title_sim * 0.45 - - # Artist similarity (weight 0.15) - sf_artist = sf.get('artist') or '' - if sf_artist and sp_artist_name: - artist_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sp_artist_name), - matching_engine.normalize_string(sf_artist) - ) - score += artist_sim * 0.15 - else: - score += 0.075 # neutral when no artist data - - # Track number match (weight 0.30) - if sf['track_number'] and sp_number: - if sf['track_number'] == sp_number: - score += 0.30 - elif abs(sf['track_number'] - sp_number) <= 1: - score += 0.12 - - # Album tag bonus (weight 0.10) — reward files whose album tag matches - sf_album = sf.get('album') or '' - if sf_album and album_name_for_match: - album_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sf_album), - matching_engine.normalize_string(album_name_for_match) - ) - score += album_sim * 0.10 - - if score > best_score and score >= 0.4: - best_score = score - best_match = i - - if best_match is not None: - used_files.add(best_match) - matches.append({ - 'spotify_track': { - 'name': sp_name, - 'track_number': sp_number, - 'disc_number': sp_disc, - 'duration_ms': sp_track.get('duration_ms', 0), - 'id': sp_track.get('id', ''), - 'artists': [a['name'] for a in sp_track.get('artists', [])], - 'uri': sp_track.get('uri', '') - }, - 'staging_file': staging_files[best_match], - 'confidence': round(best_score, 2) - }) - else: - matches.append({ - 'spotify_track': { - 'name': sp_name, - 'track_number': sp_number, - 'disc_number': sp_disc, - 'duration_ms': sp_track.get('duration_ms', 0), - 'id': sp_track.get('id', ''), - 'artists': [a['name'] for a in sp_track.get('artists', [])], - 'uri': sp_track.get('uri', '') - }, - 'staging_file': None, - 'confidence': 0 - }) - - # Unmatched staging files - unmatched_files = [sf for i, sf in enumerate(staging_files) if i not in used_files] - - return jsonify({ - 'success': True, - 'album': album_info, - 'matches': matches, - 'unmatched_files': unmatched_files - }) + payload = build_album_import_match_payload( + album_id, + album_name=album_name, + album_artist=album_artist, + file_paths=filter_file_paths, + source=source or None, + ) + return jsonify(payload) except Exception as e: logger.error(f"Error matching album for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52426,7 +48105,7 @@ def import_album_match(): def import_album_process(): """Process matched album files through the post-processing pipeline.""" try: - data = request.get_json() + data = request.get_json() or {} album = data.get('album', {}) matches = data.get('matches', []) @@ -52435,34 +48114,25 @@ def import_album_process(): processed = 0 errors = [] - album_name = album.get('name', 'Unknown Album') - artist_name = album.get('artist', 'Unknown Artist') - artist_id = album.get('artist_id', '') - album_id = album.get('id', '') + album_name = album.get('name', album.get('album_name', 'Unknown Album')) + artist_name = album.get('artist', album.get('artist_name', 'Unknown Artist')) + album_id = album.get('id', album.get('album_id', '')) + source = str(album.get('source') or data.get('source') or '').strip().lower() - # Get artist genres from Spotify if possible - artist_genres = album.get('genres', []) - if not artist_genres and artist_id and not _spotify_rate_limited(): - try: - from core.api_call_tracker import api_call_tracker - if hasattr(spotify_client, 'sp') and spotify_client.sp: - api_call_tracker.record_call('spotify', endpoint='artist') - sp_artist = spotify_client.sp.artist(artist_id) if hasattr(spotify_client, 'sp') and spotify_client.sp else None - if sp_artist: - artist_genres = sp_artist.get('genres', []) - except Exception: - pass - - # Compute total_discs across all matched tracks for multi-disc subfolder support total_discs = max( - (m['spotify_track'].get('disc_number', 1) for m in matches if m.get('spotify_track')), - default=1 + ( + match.get('track', {}).get('disc_number', 1) + for match in matches + if match.get('track') + ), + default=1, ) + artist_context = resolve_album_artist_context(album, source=source) for match in matches: staging_file = match.get('staging_file') - spotify_track = match.get('spotify_track') - if not staging_file or not spotify_track: + track = match.get('track') or {} + if not staging_file or not track: continue file_path = staging_file.get('full_path', '') @@ -52470,49 +48140,18 @@ def import_album_process(): errors.append(f"File not found: {staging_file.get('filename', '?')}") continue - track_name = spotify_track.get('name', 'Unknown Track') - track_number = spotify_track.get('track_number', 1) - disc_number = spotify_track.get('disc_number', 1) - track_artists = spotify_track.get('artists', [artist_name]) + track_name = track.get('name', 'Unknown Track') + track_number = track.get('track_number', 1) + disc_number = track.get('disc_number', 1) context_key = f"import_album_{album_id}_{track_number}_{uuid.uuid4().hex[:8]}" - context = { - 'spotify_artist': { - 'name': artist_name, - 'id': artist_id, - 'genres': artist_genres - }, - 'spotify_album': { - 'id': album_id, - 'name': album_name, - 'release_date': album.get('release_date', ''), - 'total_tracks': album.get('total_tracks', len(matches)), - 'total_discs': total_discs, - 'image_url': album.get('image_url', '') - }, - 'track_info': { - 'name': track_name, - 'id': spotify_track.get('id', ''), - 'track_number': track_number, - 'disc_number': disc_number, - 'duration_ms': spotify_track.get('duration_ms', 0), - 'artists': [{'name': a} if isinstance(a, str) else a for a in track_artists], - 'uri': spotify_track.get('uri', '') - }, - 'original_search_result': { - 'title': track_name, - 'artist': artist_name, - 'album': album_name, - 'track_number': track_number, - 'disc_number': disc_number, - 'spotify_clean_title': track_name, - 'spotify_clean_album': album_name, - 'artists': [{'name': a} if isinstance(a, str) else a for a in track_artists] - }, - 'is_album_download': True, - 'has_clean_spotify_data': True, - 'has_full_spotify_metadata': True - } + context = build_album_import_context( + album, + track, + artist_context=artist_context, + total_discs=total_discs, + source=source, + ) try: _post_process_matched_download(context_key, context, file_path) @@ -52561,35 +48200,18 @@ def import_album_process(): @app.route('/api/import/search/tracks', methods=['GET']) def import_search_tracks(): - """Search Spotify for individual tracks (used for manual singles identification).""" + """Search tracks using the configured metadata provider priority order.""" try: query = request.args.get('q', '').strip() if not query: return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 limit = min(int(request.args.get('limit', 10)), 30) + if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: + hydrabase_worker.enqueue(query, 'tracks') - if _is_hydrabase_active(): - tracks = hydrabase_client.search_tracks(query, limit=limit) - else: - if hydrabase_worker and dev_mode_enabled: - hydrabase_worker.enqueue(query, 'tracks') - tracks = spotify_client.search_tracks(query, limit=limit) - - results = [] - for t in tracks: - results.append({ - 'id': t.id, - 'name': t.name, - 'artist': ', '.join(t.artists) if hasattr(t, 'artists') and t.artists else 'Unknown Artist', - 'album': t.album if hasattr(t, 'album') else '', - 'album_id': t.album_id if hasattr(t, 'album_id') else '', - 'duration_ms': t.duration_ms if hasattr(t, 'duration_ms') else 0, - 'image_url': t.image_url if hasattr(t, 'image_url') else '', - 'track_number': t.track_number if hasattr(t, 'track_number') else 1, - }) - - return jsonify({'success': True, 'tracks': results}) + tracks = search_import_tracks(query, limit=limit) + return jsonify({'success': True, 'tracks': tracks}) except Exception as e: logger.error(f"Error searching tracks for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52616,180 +48238,51 @@ def import_singles_process(): title = file_info.get('title', '') artist = file_info.get('artist', '') - spotify_override = file_info.get('spotify_override', None) + manual_match = file_info.get('manual_match') + if manual_match is not None and not isinstance(manual_match, dict): + manual_match = None + + manual_match_source = '' + manual_match_id = None + if manual_match: + manual_match_source = str(manual_match.get('source') or '').strip().lower() + manual_match_id = str(manual_match.get('id') or '').strip() + if not manual_match_id or not manual_match_source: + errors.append(f"Malformed manual match for file: {file_info.get('filename', '?')}") + continue # Fallback to filename parsing if no metadata - if not title and not spotify_override: - parsed = _parse_filename_metadata(file_info.get('filename', '')) - title = parsed.get('title', os.path.splitext(file_info.get('filename', 'Unknown'))[0]) + if not title and not manual_match: + parsed = parse_filename_metadata(file_info.get('filename', '')) + title = parsed.get('title') or os.path.splitext(file_info.get('filename', 'Unknown'))[0] if not artist: artist = parsed.get('artist', '') - # Search Spotify for rich metadata - spotify_track_data = None - spotify_artist_data = None - spotify_album_data = None + from core.metadata_service import get_single_track_import_context - # If a manual spotify_override is provided, look up that specific track - if spotify_override and spotify_override.get('id'): - try: - override_id = spotify_override['id'] - sp_track = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='track') - sp_track = spotify_client.sp.track(override_id) - if sp_track: - sp_track_artists = sp_track.get('artists', []) - spotify_track_data = { - 'name': sp_track.get('name', ''), - 'id': override_id, - 'track_number': sp_track.get('track_number', 1), - 'disc_number': sp_track.get('disc_number', 1), - 'duration_ms': sp_track.get('duration_ms', 0), - 'artists': [{'name': a.get('name', '')} for a in sp_track_artists], - 'uri': sp_track.get('uri', f"spotify:track:{override_id}") - } - title = sp_track.get('name', title) - artist = sp_track_artists[0].get('name', artist) if sp_track_artists else artist - # Get album info - sp_album_info = sp_track.get('album', {}) - if sp_album_info: - spotify_album_data = { - 'id': sp_album_info.get('id', ''), - 'name': sp_album_info.get('name', ''), - 'release_date': sp_album_info.get('release_date', ''), - 'total_tracks': sp_album_info.get('total_tracks', 1), - 'image_url': (sp_album_info.get('images', [{}])[0].get('url') if sp_album_info.get('images') else ''), - 'album_type': sp_album_info.get('album_type', 'album'), - } - album_artists = sp_album_info.get('artists', []) - if album_artists: - spotify_artist_data = { - 'name': album_artists[0].get('name', artist), - 'id': album_artists[0].get('id', ''), - 'genres': [] - } - try: - sp_a = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='artist') - sp_a = spotify_client.sp.artist(album_artists[0]['id']) - if sp_a: - spotify_artist_data['genres'] = sp_a.get('genres', []) - except Exception: - pass - except Exception as override_err: - logger.warning(f"Spotify override lookup failed for track {spotify_override.get('id')}: {override_err}") - - if not spotify_track_data and title: - try: - search_q = f"{title} {artist}" if artist else title - tracks = spotify_client.search_tracks(search_q, limit=1) - if tracks: - t = tracks[0] - spotify_track_data = { - 'name': t.name, - 'id': t.id, - 'track_number': t.track_number if hasattr(t, 'track_number') else 1, - 'disc_number': 1, - 'duration_ms': t.duration_ms if hasattr(t, 'duration_ms') else 0, - 'artists': [{'name': a} for a in (t.artists if hasattr(t, 'artists') else [artist])], - 'uri': f"spotify:track:{t.id}" - } - # Get album info from the track's album - if hasattr(t, 'album_id') and t.album_id: - sp_album = spotify_client.get_album(t.album_id) - if sp_album: - spotify_album_data = { - 'id': t.album_id, - 'name': sp_album.get('name', ''), - 'release_date': sp_album.get('release_date', ''), - 'total_tracks': sp_album.get('total_tracks', 1), - 'image_url': (sp_album.get('images', [{}])[0].get('url') if sp_album.get('images') else ''), - 'album_type': sp_album.get('album_type', 'album'), - } - # Get artist genres - sp_artists = sp_album.get('artists', []) - if sp_artists: - spotify_artist_data = { - 'name': sp_artists[0].get('name', artist), - 'id': sp_artists[0].get('id', ''), - 'genres': [] - } - try: - sp_a = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='artist') - sp_a = spotify_client.sp.artist(sp_artists[0]['id']) - if sp_a: - spotify_artist_data['genres'] = sp_a.get('genres', []) - except Exception: - pass - - # Fallback artist data from track - if not spotify_artist_data: - track_artists = t.artists if hasattr(t, 'artists') else [artist] - spotify_artist_data = { - 'name': track_artists[0] if track_artists else artist, - 'id': '', - 'genres': [] - } - - # Fallback album data - if not spotify_album_data: - spotify_album_data = { - 'id': '', - 'name': t.album if hasattr(t, 'album') else '', - 'release_date': '', - 'total_tracks': 1, - 'image_url': t.image_url if hasattr(t, 'image_url') else '', - 'album_type': t.album_type if hasattr(t, 'album_type') else 'album', - } - except Exception as sp_err: - logger.warning(f"Spotify lookup failed for '{title}': {sp_err}") - - # Build context — use Spotify data if found, else use file metadata - if not spotify_artist_data: - spotify_artist_data = {'name': artist or 'Unknown Artist', 'id': '', 'genres': []} - if not spotify_album_data: - spotify_album_data = {'id': '', 'name': '', 'release_date': '', 'total_tracks': 1, 'image_url': '', 'album_type': 'album'} - if not spotify_track_data: - spotify_track_data = { - 'name': title, 'id': '', 'track_number': 1, 'disc_number': 1, - 'duration_ms': 0, 'artists': [{'name': artist or 'Unknown Artist'}], 'uri': '' - } - - final_title = spotify_track_data.get('name', title) - final_artist = spotify_artist_data.get('name', artist) - final_album = spotify_album_data.get('name', '') + resolved = get_single_track_import_context( + title, + artist, + override_id=manual_match_id, + override_source=manual_match_source, + ) + context = normalize_import_context(resolved['context']) + artist_data = get_import_context_artist(context) + track_data = get_import_track_info(context) + final_title = track_data.get('name', title) + final_artist = artist_data.get('name', artist) context_key = f"import_single_{uuid.uuid4().hex[:8]}" - context = { - 'spotify_artist': spotify_artist_data, - 'spotify_album': spotify_album_data, - 'track_info': spotify_track_data, - 'original_search_result': { - 'title': final_title, - 'artist': final_artist, - 'album': final_album, - 'track_number': spotify_track_data.get('track_number', 1), - 'disc_number': 1, - 'spotify_clean_title': final_title, - 'spotify_clean_album': final_album, - 'artists': spotify_track_data.get('artists', [{'name': final_artist}]) - }, - 'is_album_download': False, - 'has_clean_spotify_data': bool(spotify_track_data.get('id')), - 'has_full_spotify_metadata': bool(spotify_track_data.get('id')) - } try: _post_process_matched_download(context_key, context, file_path) processed += 1 - logger.info(f"Import single processed: {final_title} by {final_artist}") + logger.info( + "Import single processed: %s by %s (source=%s)", + final_title, + final_artist, + resolved.get('source') or 'local', + ) except Exception as proc_err: err_msg = f"{title}: {str(proc_err)}" errors.append(err_msg) @@ -52830,126 +48323,6 @@ def import_singles_process(): logger.error(f"Error processing singles import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 -# --- Import Suggestion Cache (server-side background builder) --- - -_import_suggestions_cache = { - 'suggestions': [], - 'building': False, - 'built': False, -} - -def _build_import_suggestions_background(): - """Background thread: extract hints from staging folder, search Spotify, cache results.""" - cache = _import_suggestions_cache - if cache['building']: - return - cache['building'] = True - - try: - staging_path = _get_staging_path() - if not os.path.isdir(staging_path): - cache['suggestions'] = [] - cache['built'] = True - cache['building'] = False - return - - # Reuse the hint extraction logic - tag_albums = {} - folder_hints = {} - - for root, _dirs, filenames in os.walk(staging_path): - audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] - if not audio_files: - continue - rel_dir = os.path.relpath(root, staging_path) - if rel_dir != '.': - top_folder = rel_dir.split(os.sep)[0] - folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) - for fname in audio_files: - full_path = os.path.join(root, fname) - try: - from mutagen import File as MutagenFile - tags = MutagenFile(full_path, easy=True) - if tags: - album = (tags.get('album') or [None])[0] - artist = (tags.get('artist') or (tags.get('albumartist') or [None]))[0] - if album: - key = (album.strip(), (artist or '').strip()) - tag_albums[key] = tag_albums.get(key, 0) + 1 - except Exception: - pass - - queries = [] - seen_lower = set() - for (album, artist), _ in sorted(tag_albums.items(), key=lambda x: -x[1]): - q = f"{album} {artist}".strip() if artist else album - if q.lower() not in seen_lower: - seen_lower.add(q.lower()) - queries.append(q) - for folder, _ in sorted(folder_hints.items(), key=lambda x: -x[1]): - q = folder.replace('_', ' ') - if q.lower() not in seen_lower: - seen_lower.add(q.lower()) - queries.append(q) - queries = queries[:5] - - if not queries: - cache['suggestions'] = [] - cache['built'] = True - cache['building'] = False - return - - suggestions = [] - seen_ids = set() - for q in queries: - try: - if not spotify_client: - break - albums = spotify_client.search_albums(q, limit=2) - for a in albums: - if a.id not in seen_ids: - seen_ids.add(a.id) - suggestions.append({ - 'id': a.id, - 'name': a.name, - 'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist', - 'release_date': a.release_date or '', - 'total_tracks': a.total_tracks, - 'image_url': a.image_url, - 'album_type': a.album_type or 'album', - }) - except Exception as e: - logger.warning(f"Import suggestion search failed for '{q}': {e}") - - cache['suggestions'] = suggestions[:8] - cache['built'] = True - logger.info(f"Import suggestions cache built: {len(cache['suggestions'])} suggestions from {len(queries)} hints") - except Exception as e: - logger.error(f"Error building import suggestions cache: {e}") - cache['suggestions'] = [] - cache['built'] = True - finally: - cache['building'] = False - - -def start_import_suggestions_cache(): - """Start building the import suggestions cache in a background thread (called on server startup).""" - threading.Thread( - target=_build_import_suggestions_background, - daemon=True, - name='import-suggestions-cache' - ).start() - - -def refresh_import_suggestions_cache(): - """Invalidate and rebuild the suggestions cache (called after imports change staging contents).""" - _import_suggestions_cache['built'] = False - threading.Thread( - target=_build_import_suggestions_background, - daemon=True, - name='import-suggestions-cache' - ).start() - # ── Auto-Import Worker ── auto_import_worker = None @@ -53089,7 +48462,7 @@ def auto_import_clear_completed(): @app.route('/api/import/staging/suggestions', methods=['GET']) def import_staging_suggestions(): """Return cached import suggestions. If cache isn't built yet, returns partial/empty with a flag.""" - cache = _import_suggestions_cache + cache = get_import_suggestions_cache() return jsonify({ 'success': True, 'suggestions': cache['suggestions'],